From 842058416efac17284bbefd144dde4d592de6cbb Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Tue, 8 Sep 2026 17:28:38 -0400 Subject: [PATCH 1/7] Add Socket Firewall to Emulate workflows --- .github/workflows/ci.yml | 21 +++++++++++++++++++++ .github/workflows/lint.yml | 12 ++++++++++++ .github/workflows/release.yml | 21 +++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec1259a..049d243 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,11 +5,23 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Socket Firewall + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + configure-bun: true - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: @@ -86,6 +98,15 @@ jobs: node: [22, 24] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Socket Firewall + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + configure-bun: true - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d31a30d..94fb201 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,11 +5,23 @@ on: branches: [main] pull_request: +permissions: + contents: read + jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup Socket Firewall + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + configure-bun: true - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6fe1bf0..5f00f27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,6 +11,9 @@ on: type: string required: true secrets: + PUBLIC_SOCKET_FIREWALL_TOKEN: + description: Public Socket Firewall token for release dependency downloads + required: false SDK_BOT_PRIVATE_KEY: description: Private key for the org SDK bot GitHub App, used to mint a short-lived token scoped to workos/homebrew-tap required: false @@ -56,6 +59,12 @@ jobs: with: ref: ${{ inputs.tag_name }} + - name: Setup Socket Firewall + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + configure-bun: true + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 with: bun-version: 1.3.14 @@ -73,6 +82,9 @@ jobs: ./src/cli.ts --outfile "dist/${{ matrix.asset }}" + - name: Teardown Socket Firewall + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.asset }} @@ -202,9 +214,18 @@ jobs: node-version: 24 registry-url: 'https://registry.npmjs.org' + - name: Setup Socket Firewall + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + configure-bun: true + - name: Install run: bun install --frozen-lockfile + - name: Teardown Socket Firewall + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d + - name: Build run: bun run build From c7b598b45d67356d6c3c56736a618238a5a9d0c0 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Tue, 8 Sep 2026 17:33:31 -0400 Subject: [PATCH 2/7] Cover Bun cross-compile downloads --- .github/workflows/release.yml | 4 +++ scripts/build-binaries.sh | 48 +++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5f00f27..5b6c923 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,6 +59,10 @@ jobs: with: ref: ${{ inputs.tag_name }} + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + - name: Setup Socket Firewall uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d with: diff --git a/scripts/build-binaries.sh b/scripts/build-binaries.sh index d1df9e7..b73ad5d 100755 --- a/scripts/build-binaries.sh +++ b/scripts/build-binaries.sh @@ -17,11 +17,59 @@ TARGETS=( "bun-windows-arm64:workos-emulate-windows-arm64.exe" ) +COMPILE_TARGETS_DIR="${BUN_COMPILE_TARGETS_DIR:-.bun-compile-targets}" +rm -rf "$COMPILE_TARGETS_DIR" +mkdir -p "$COMPILE_TARGETS_DIR" + +compile_package_for_target() { + case "$1" in + bun-darwin-arm64) printf '%s\n' '@oven/bun-darwin-aarch64' ;; + bun-darwin-x64-baseline) printf '%s\n' '@oven/bun-darwin-x64-baseline' ;; + bun-linux-x64-baseline) printf '%s\n' '@oven/bun-linux-x64-baseline' ;; + bun-linux-arm64) printf '%s\n' '@oven/bun-linux-aarch64' ;; + bun-linux-x64-musl-baseline) printf '%s\n' '@oven/bun-linux-x64-musl-baseline' ;; + bun-linux-arm64-musl) printf '%s\n' '@oven/bun-linux-aarch64-musl' ;; + bun-windows-x64-baseline) printf '%s\n' '@oven/bun-windows-x64-baseline' ;; + bun-windows-arm64) printf '%s\n' '@oven/bun-windows-aarch64' ;; + *) + echo "Unsupported Bun compile target: $1" >&2 + return 1 + ;; + esac +} + +compile_binary_for_target() { + case "$1" in + bun-windows-*) printf '%s\n' 'bun.exe' ;; + *) printf '%s\n' 'bun' ;; + esac +} + +unpack_compile_target() { + local target="$1" + local package="$2" + local destination="$COMPILE_TARGETS_DIR/$target" + local tarball + + mkdir -p "$destination" + npm pack --pack-destination "$destination" "${package}@$(bun --version)" + tarball="$(find "$destination" -maxdepth 1 -name '*.tgz' -print -quit)" + test -n "$tarball" + tar -xzf "$tarball" -C "$destination" +} + for entry in "${TARGETS[@]}"; do target="${entry%%:*}" artifact="${entry#*:}" + compile_package="$(compile_package_for_target "$target")" + compile_binary="$(compile_binary_for_target "$target")" + compile_path="$COMPILE_TARGETS_DIR/$target/package/bin/$compile_binary" + + unpack_compile_target "$target" "$compile_package" + test -f "$compile_path" bun build \ --compile \ + --compile-executable-path "$compile_path" \ --no-compile-autoload-dotenv \ --no-compile-autoload-bunfig \ --target="$target" \ From 9a58f38c3ff4ca61c5db00e9ade7bea15f379f24 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Wed, 9 Sep 2026 09:36:32 -0400 Subject: [PATCH 3/7] Protect Docker Bun installs with BuildKit SFW config --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++- .github/workflows/release.yml | 33 ++++++++++++---- Dockerfile | 29 +++++++++++++- scripts/build-binaries.sh | 44 ++++++++++++++++++--- 4 files changed, 160 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 049d243..2f66d7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -130,13 +130,47 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + - name: Setup Socket Firewall + id: sfw + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + env: + XDG_CONFIG_HOME: ${{ runner.temp }}/sfw-docker-bun + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + configure-bun: true + + - name: Build image + if: steps.sfw.outputs.active == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: false load: true tags: emulate:ci + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + secret-files: | + sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + + - name: Build image (external fork fallback) + if: steps.sfw.outputs.active == 'false' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: false + load: true + tags: emulate:ci + + - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - name: Container smoke test shell: bash @@ -156,13 +190,47 @@ jobs: runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + - name: Setup Socket Firewall + id: sfw + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + env: + XDG_CONFIG_HOME: ${{ runner.temp }}/sfw-docker-bun + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + configure-bun: true + + - name: Build image + if: steps.sfw.outputs.active == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: false load: true tags: emulate:ci + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + secret-files: | + sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + + - name: Build image (external fork fallback) + if: steps.sfw.outputs.active == 'false' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: false + load: true + tags: emulate:ci + + - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - name: Container smoke test shell: bash diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5b6c923..71034ce 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,6 +64,7 @@ jobs: node-version: 22 - name: Setup Socket Firewall + id: sfw uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d with: token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} @@ -77,16 +78,10 @@ jobs: run: bun install --frozen-lockfile - name: Build standalone binary - run: >- - bun build - --compile - --no-compile-autoload-dotenv - --no-compile-autoload-bunfig - --target="${{ matrix.target }}" - ./src/cli.ts - --outfile "dist/${{ matrix.asset }}" + run: scripts/build-binaries.sh "${{ inputs.tag_name }}" "${{ matrix.target }}" "${{ matrix.asset }}" - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -219,6 +214,7 @@ jobs: registry-url: 'https://registry.npmjs.org' - name: Setup Socket Firewall + id: sfw uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d with: token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} @@ -228,6 +224,7 @@ jobs: run: bun install --frozen-lockfile - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - name: Build @@ -304,12 +301,32 @@ jobs: - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0 - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - name: Setup Socket Firewall + id: sfw + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d + env: + XDG_CONFIG_HOME: ${{ runner.temp }}/sfw-docker-bun + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + configure-bun: true + - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.tags.outputs.tags }} + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + secret-files: | + sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + + - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d homebrew: name: Update Homebrew formula diff --git a/Dockerfile b/Dockerfile index d3424bd..bf4d071 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ # syntax=docker/dockerfile:1 +# CI/release callers pass Bun's Socket Firewall config as a BuildKit secret and +# set SFW_REQUIRED=true. Local Docker builds omit both and keep using Bun's +# normal public-registry configuration. +ARG SFW_REQUIRED=false + # Build stage: compile TypeScript to dist/ from the bun lockfile. # bun.lock pins tree-sitter-kotlin (a transitive devDep via @workos/openapi-spec # -> @workos/oagen) to a git+ssh URL that can't clone inside the image without @@ -9,9 +14,19 @@ # other dependency — including typescript@5.9.3 — stays at its locked version, # keeping the build reproducible. FROM oven/bun:1.3.14 AS builder +ARG SFW_REQUIRED WORKDIR /app COPY package.json bun.lock ./ -RUN sed -i \ +RUN --mount=type=secret,id=sfw_bunfig,target=/run/secrets/.bunfig.toml \ + set -eu; \ + if [ "${SFW_REQUIRED:-false}" = "true" ] && [ ! -s /run/secrets/.bunfig.toml ]; then \ + echo "Socket Firewall Bun config secret is required for Docker dependency installs." >&2; \ + exit 1; \ + fi; \ + if [ -s /run/secrets/.bunfig.toml ]; then \ + export XDG_CONFIG_HOME=/run/secrets; \ + fi; \ + sed -i \ -e 's|git+ssh://git@github.com/fwcd/tree-sitter-kotlin.git#|github:fwcd/tree-sitter-kotlin#|g' \ -e 's/"sha512-onbog[^"]*"/""/g' \ bun.lock \ @@ -24,9 +39,19 @@ RUN bun run build # This stage only needs production deps (no git+ssh transitive devDeps), so # the bun lockfile works correctly. FROM oven/bun:1.3.14 AS deps +ARG SFW_REQUIRED WORKDIR /app COPY package.json bun.lock ./ -RUN bun install --frozen-lockfile --production --ignore-scripts +RUN --mount=type=secret,id=sfw_bunfig,target=/run/secrets/.bunfig.toml \ + set -eu; \ + if [ "${SFW_REQUIRED:-false}" = "true" ] && [ ! -s /run/secrets/.bunfig.toml ]; then \ + echo "Socket Firewall Bun config secret is required for Docker dependency installs." >&2; \ + exit 1; \ + fi; \ + if [ -s /run/secrets/.bunfig.toml ]; then \ + export XDG_CONFIG_HOME=/run/secrets; \ + fi; \ + bun install --frozen-lockfile --production --ignore-scripts # Runtime stage: minimal Node image with only what the emulator needs. FROM node:22-alpine diff --git a/scripts/build-binaries.sh b/scripts/build-binaries.sh index b73ad5d..202440f 100755 --- a/scripts/build-binaries.sh +++ b/scripts/build-binaries.sh @@ -1,9 +1,27 @@ #!/usr/bin/env bash set -euo pipefail -VERSION="${1:?usage: build-binaries.sh }" -OUT="dist/binaries" -rm -rf "$OUT" && mkdir -p "$OUT" +usage() { + echo "usage: build-binaries.sh [target asset]" >&2 + exit 2 +} + +VERSION="${1:-}" +[[ -n "$VERSION" ]] || usage +REQUESTED_TARGET="${2:-}" +REQUESTED_ASSET="${3:-}" +if [[ -n "$REQUESTED_TARGET" && -z "$REQUESTED_ASSET" ]] || [[ -z "$REQUESTED_TARGET" && -n "$REQUESTED_ASSET" ]]; then + usage +fi + +if [[ -n "$REQUESTED_TARGET" ]]; then + OUT="${BUN_BINARY_OUT_DIR:-dist}" + mkdir -p "$OUT" + rm -f "$OUT/$REQUESTED_ASSET" +else + OUT="${BUN_BINARY_OUT_DIR:-dist/binaries}" + rm -rf "$OUT" && mkdir -p "$OUT" +fi # target:artifact pairs (plain array — macOS /bin/bash 3.2 has no associative arrays) TARGETS=( @@ -58,9 +76,11 @@ unpack_compile_target() { tar -xzf "$tarball" -C "$destination" } -for entry in "${TARGETS[@]}"; do - target="${entry%%:*}" - artifact="${entry#*:}" +build_target() { + local target="$1" + local artifact="$2" + local compile_package compile_binary compile_path + compile_package="$(compile_package_for_target "$target")" compile_binary="$(compile_binary_for_target "$target")" compile_path="$COMPILE_TARGETS_DIR/$target/package/bin/$compile_binary" @@ -75,6 +95,18 @@ for entry in "${TARGETS[@]}"; do --target="$target" \ ./src/cli.ts \ --outfile "$OUT/$artifact" +} + +if [[ -n "$REQUESTED_TARGET" ]]; then + build_target "$REQUESTED_TARGET" "$REQUESTED_ASSET" + echo "Built $REQUESTED_ASSET for $VERSION" + exit 0 +fi + +for entry in "${TARGETS[@]}"; do + target="${entry%%:*}" + artifact="${entry#*:}" + build_target "$target" "$artifact" done (cd "$OUT" && shasum -a 256 workos-emulate-* > checksums.txt) From 391f94fe63474b9e2c414e226368f23c0040a1ce Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Wed, 9 Sep 2026 10:22:43 -0400 Subject: [PATCH 4/7] Add pre-publication Docker image secret scans --- .github/workflows/ci.yml | 108 +++- .github/workflows/release.yml | 53 +- scripts/create-sfw-image-scan-needles.py | 104 ++++ scripts/scan-image-secrets.py | 664 +++++++++++++++++++++++ 4 files changed, 917 insertions(+), 12 deletions(-) create mode 100755 scripts/create-sfw-image-scan-needles.py create mode 100755 scripts/scan-image-secrets.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f66d7b..bbda39f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,13 +143,24 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Build image + - name: Test image secret scanner + run: python3 scripts/scan-image-secrets.py --self-test + + - name: Create private image scan needles + if: steps.sfw.outputs.active == 'true' + env: + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/create-sfw-image-scan-needles.py + + - name: Build authenticated image for scan if: steps.sfw.outputs.active == 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: false load: true + platforms: linux/amd64 tags: emulate:ci build-args: | SFW_REQUIRED=true @@ -159,6 +170,43 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + - name: Scan authenticated image + if: steps.sfw.outputs.active == 'true' + env: + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/scan-image-secrets.py --docker-image emulate:ci --needles-file "$SFW_IMAGE_SCAN_NEEDLES" + + - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d + + - name: Remove private Docker scan materials + if: ${{ always() }} + env: + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: | + rm -f "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" + if [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker scan material was not removed." + exit 1 + fi + + - name: Load scanned image from clean cache (publication disabled) + if: steps.sfw.outputs.active == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: false + load: true + platforms: linux/amd64 + tags: emulate:ci + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + - name: Build image (external fork fallback) if: steps.sfw.outputs.active == 'false' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -168,10 +216,6 @@ jobs: load: true tags: emulate:ci - - name: Teardown Socket Firewall - if: ${{ always() && steps.sfw.outputs.active == 'true' }} - uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - - name: Container smoke test shell: bash run: | @@ -203,13 +247,24 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Build image + - name: Test image secret scanner + run: python3 scripts/scan-image-secrets.py --self-test + + - name: Create private image scan needles + if: steps.sfw.outputs.active == 'true' + env: + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/create-sfw-image-scan-needles.py + + - name: Build authenticated image for scan if: steps.sfw.outputs.active == 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . push: false load: true + platforms: linux/arm64 tags: emulate:ci build-args: | SFW_REQUIRED=true @@ -219,6 +274,43 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + - name: Scan authenticated image + if: steps.sfw.outputs.active == 'true' + env: + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/scan-image-secrets.py --docker-image emulate:ci --needles-file "$SFW_IMAGE_SCAN_NEEDLES" + + - name: Teardown Socket Firewall + if: ${{ always() && steps.sfw.outputs.active == 'true' }} + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d + + - name: Remove private Docker scan materials + if: ${{ always() }} + env: + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: | + rm -f "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" + if [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker scan material was not removed." + exit 1 + fi + + - name: Load scanned image from clean cache (publication disabled) + if: steps.sfw.outputs.active == 'true' + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + push: false + load: true + platforms: linux/arm64 + tags: emulate:ci + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + - name: Build image (external fork fallback) if: steps.sfw.outputs.active == 'false' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -228,10 +320,6 @@ jobs: load: true tags: emulate:ci - - name: Teardown Socket Firewall - if: ${{ always() && steps.sfw.outputs.active == 'true' }} - uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - - name: Container smoke test shell: bash run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71034ce..b4fcf30 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -310,11 +310,28 @@ jobs: token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} configure-bun: true - - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + - name: Require Socket Firewall for container publication + if: steps.sfw.outputs.active != 'true' + run: | + echo "::error::Socket Firewall must be active before container publication." + exit 1 + + - name: Test image secret scanner + run: python3 scripts/scan-image-secrets.py --self-test + + - name: Create private image scan needles + env: + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/create-sfw-image-scan-needles.py + + - name: Build authenticated image archive for scan + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . platforms: linux/amd64,linux/arm64 - push: true + push: false + outputs: type=oci,dest=${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar tags: ${{ steps.tags.outputs.tags }} build-args: | SFW_REQUIRED=true @@ -324,10 +341,42 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + - name: Scan authenticated image archive + env: + EMULATE_IMAGE_ARCHIVE: ${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: python3 scripts/scan-image-secrets.py --oci-archive "$EMULATE_IMAGE_ARCHIVE" --needles-file "$SFW_IMAGE_SCAN_NEEDLES" + - name: Teardown Socket Firewall if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d + - name: Remove private Docker scan materials + if: ${{ always() }} + env: + EMULATE_IMAGE_ARCHIVE: ${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar + SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml + SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json + run: | + rm -f "$EMULATE_IMAGE_ARCHIVE" "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" + if [ -e "$EMULATE_IMAGE_ARCHIVE" ] || [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker scan material was not removed." + exit 1 + fi + + - name: Publish scanned image from clean cache + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.tags.outputs.tags }} + build-args: | + SFW_REQUIRED=true + add-hosts: | + registry.npmjs.org:127.0.0.1 + registry.yarnpkg.com:127.0.0.1 + homebrew: name: Update Homebrew formula runs-on: macos-15 diff --git a/scripts/create-sfw-image-scan-needles.py b/scripts/create-sfw-image-scan-needles.py new file mode 100755 index 0000000..c33c580 --- /dev/null +++ b/scripts/create-sfw-image-scan-needles.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Create a private needle file for Docker image leak scans. + +The input is the temporary Bun config produced by the pinned Socket Firewall +setup action. The output may contain credentials and must stay in runner temp. +This script intentionally prints only counts, never paths or values. +""" + +from __future__ import annotations + +import base64 +import json +import os +import re +import stat +import sys +from pathlib import Path + +MIN_NEEDLE_LENGTH = 8 +TOKEN_RE = re.compile(rb"[A-Za-z0-9][A-Za-z0-9+/_:.,=@%?&#$~-]{7,}") +SENSITIVE_KEY_RE = re.compile( + rb"(?i)(auth|token|password|credential|secret|registry|socket|npm|bun)" +) + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def add_needle(needles: set[bytes], value: bytes) -> None: + value = value.strip() + if len(value) < MIN_NEEDLE_LENGTH: + return + if not any(byte > 0x20 for byte in value): + return + needles.add(value) + + +def derive_needles(config_bytes: bytes) -> list[bytes]: + needles: set[bytes] = set() + stripped_config = config_bytes.strip() + add_needle(needles, stripped_config) + + for raw_line in config_bytes.splitlines(): + line = raw_line.strip() + if not line or line.startswith(b"#"): + continue + + if SENSITIVE_KEY_RE.search(line): + add_needle(needles, line) + # Capture common key/value forms without needing to know the exact + # config format emitted by the action. + for separator in (b"=", b":"): + if separator in line: + add_needle(needles, line.split(separator, 1)[1].strip().strip(b"'\"")) + + for token in TOKEN_RE.findall(line): + if b"socket" in token.lower() or SENSITIVE_KEY_RE.search(line): + add_needle(needles, token.strip(b"'\"")) + + return sorted(needles, key=lambda item: (len(item), item)) + + +def main() -> int: + config_path = os.environ.get("SFW_BUN_CONFIG_FILE") + output_path = os.environ.get("SFW_IMAGE_SCAN_NEEDLES") + if not config_path: + fail("SFW_BUN_CONFIG_FILE is required") + if not output_path: + fail("SFW_IMAGE_SCAN_NEEDLES is required") + + config = Path(config_path) + output = Path(output_path) + try: + config_bytes = config.read_bytes() + except FileNotFoundError: + fail("Socket Firewall config was not created") + + if not config_bytes.strip(): + fail("Socket Firewall config is empty") + + needles = derive_needles(config_bytes) + if not needles: + fail("no scan needles could be derived from the Socket Firewall config") + + payload = { + "schemaVersion": 1, + "needles": [base64.b64encode(needle).decode("ascii") for needle in needles], + } + + output.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, separators=(",", ":")) + handle.write("\n") + os.chmod(output, stat.S_IRUSR | stat.S_IWUSR) + + print(f"created private Docker image scan needles: count={len(needles)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/scan-image-secrets.py b/scripts/scan-image-secrets.py new file mode 100755 index 0000000..f604ce2 --- /dev/null +++ b/scripts/scan-image-secrets.py @@ -0,0 +1,664 @@ +#!/usr/bin/env python3 +"""Fail-closed Docker/OCI image scanner for ephemeral install config leaks. + +The scanner inspects image config/history metadata and every layer tar member, +including deleted lower-layer files. Diagnostics intentionally redact matched +values, filenames, and archive member paths. +""" + +from __future__ import annotations + +import argparse +import base64 +import gzip +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable + +DOCKER_ARCHIVE_MANIFEST = "manifest.json" +OCI_LAYOUT = "oci-layout" +OCI_INDEX = "index.json" + +INDEX_MEDIA_TYPES = { + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", +} +MANIFEST_MEDIA_TYPES = { + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +} +CONFIG_MEDIA_TYPES = { + "application/vnd.oci.image.config.v1+json", + "application/vnd.docker.container.image.v1+json", +} +TAR_LAYER_MEDIA_TYPES = { + "application/vnd.oci.image.layer.v1.tar", + "application/vnd.oci.image.layer.v1.tar+gzip", + "application/vnd.oci.image.layer.nondistributable.v1.tar", + "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "application/vnd.docker.image.rootfs.diff.tar", + "application/vnd.docker.image.rootfs.diff.tar.gzip", + "application/vnd.docker.image.rootfs.foreign.diff.tar", + "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", +} +JSON_BLOB_MEDIA_TYPES = { + "application/vnd.in-toto+json", + "application/vnd.dsse.envelope.v1+json", + "application/vnd.oci.empty.v1+json", +} + +# These catch committed package-manager config files or credential declarations +# even if the dynamic credential value itself is transformed. They are not +# printed when matched. +FORBIDDEN_PATH_NAMES = { + ".bunfig.toml", + "bunfig.toml", +} +FORBIDDEN_CONTENT_MARKERS = [ + b"PUBLIC_SOCKET_FIREWALL_TOKEN", + b"SOCKET_FIREWALL_TOKEN", +] + + +class ScanError(RuntimeError): + """Raised when an archive cannot be fully inspected.""" + + +@dataclass +class Stats: + images: int = 0 + layers: int = 0 + paths: int = 0 + metadata: int = 0 + blobs: int = 0 + hits: dict[str, int] = field(default_factory=dict) + + def hit(self, kind: str) -> None: + self.hits[kind] = self.hits.get(kind, 0) + 1 + + @property + def hit_count(self) -> int: + return sum(self.hits.values()) + + def merge(self, other: "Stats") -> None: + self.images += other.images + self.layers += other.layers + self.paths += other.paths + self.metadata += other.metadata + self.blobs += other.blobs + for kind, count in other.hits.items(): + self.hits[kind] = self.hits.get(kind, 0) + count + + +def load_needles(path: Path) -> list[bytes]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception as error: # noqa: BLE001 - fail closed with concise cause + raise ScanError("needle file is not valid JSON") from error + + encoded = payload.get("needles") + if payload.get("schemaVersion") != 1 or not isinstance(encoded, list): + raise ScanError("needle file schema is invalid") + + needles: list[bytes] = [] + for item in encoded: + if not isinstance(item, str): + raise ScanError("needle file contains a non-string entry") + try: + value = base64.b64decode(item, validate=True) + except Exception as error: # noqa: BLE001 + raise ScanError("needle file contains invalid base64") from error + if len(value) >= 8: + needles.append(value) + + # Deduplicate while preserving deterministic order. + seen: set[bytes] = set() + unique = [] + for needle in needles: + if needle not in seen: + unique.append(needle) + seen.add(needle) + + if not unique: + raise ScanError("needle file contained no usable scan needles") + return unique + + +def check_bytes(data: bytes, needles: Iterable[bytes], stats: Stats, location: str) -> None: + del location # Locations are intentionally redacted from diagnostics. + lower_data = data.lower() + for needle in needles: + if needle and needle in data: + stats.hit("dynamic-needle") + for marker in FORBIDDEN_CONTENT_MARKERS: + if marker.lower() in lower_data: + stats.hit("credential-marker") + + +def check_path(name: str, needles: Iterable[bytes], stats: Stats) -> None: + encoded = name.encode("utf-8", "surrogateescape") + for part in name.replace("\\", "/").split("/"): + normalized = part[4:] if part.startswith(".wh.") else part + if normalized in FORBIDDEN_PATH_NAMES: + stats.hit("forbidden-package-config-path") + for needle in needles: + if needle and needle in encoded: + stats.hit("dynamic-needle-in-path") + + +def safe_tar_members(tar: tarfile.TarFile) -> dict[str, tarfile.TarInfo]: + members: dict[str, tarfile.TarInfo] = {} + for member in tar.getmembers(): + name = member.name + normalized = Path(name) + if normalized.is_absolute() or ".." in normalized.parts: + raise ScanError("archive contains an unsafe member path") + if name in members: + raise ScanError("archive contains duplicate member paths") + members[name] = member + return members + + +def read_tar_member(tar: tarfile.TarFile, member: tarfile.TarInfo) -> bytes: + if member.size > 1024 * 1024 * 1024: + raise ScanError("archive member is too large to inspect") + fileobj = tar.extractfile(member) + if fileobj is None: + raise ScanError("archive member could not be read") + return fileobj.read() + + +def open_archive(path: Path) -> tuple[tarfile.TarFile, dict[str, tarfile.TarInfo]]: + try: + tar = tarfile.open(path, "r:*") + except tarfile.TarError as error: + raise ScanError("image archive is not a readable tar archive") from error + return tar, safe_tar_members(tar) + + +def read_named_member( + tar: tarfile.TarFile, members: dict[str, tarfile.TarInfo], name: str +) -> bytes: + member = members.get(name) + if member is None: + raise ScanError("referenced archive member is missing") + if not member.isfile(): + raise ScanError("referenced archive member is not a file") + return read_tar_member(tar, member) + + +def parse_json(data: bytes, description: str) -> Any: + try: + return json.loads(data.decode("utf-8")) + except Exception as error: # noqa: BLE001 + raise ScanError(f"{description} is not valid JSON") from error + + +def maybe_decompress_layer(data: bytes, media_type: str | None) -> bytes: + if media_type and media_type.endswith("+zstd"): + raise ScanError("zstd-compressed layers are not supported") + if media_type and "gzip" in media_type: + try: + return gzip.decompress(data) + except Exception as error: # noqa: BLE001 + raise ScanError("gzip layer could not be decompressed") from error + if data.startswith(b"\x1f\x8b"): + try: + return gzip.decompress(data) + except Exception as error: # noqa: BLE001 + raise ScanError("gzip layer could not be decompressed") from error + return data + + +def scan_layer_bytes( + data: bytes, + needles: Iterable[bytes], + stats: Stats, + media_type: str | None = None, +) -> None: + layer = maybe_decompress_layer(data, media_type) + stats.layers += 1 + check_bytes(layer, needles, stats, "layer-tar") + + try: + with tarfile.open(fileobj=io.BytesIO(layer), mode="r:") as layer_tar: + for member in layer_tar: + stats.paths += 1 + check_path(member.name, needles, stats) + if member.linkname: + check_path(member.linkname, needles, stats) + if member.isfile(): + fileobj = layer_tar.extractfile(member) + if fileobj is None: + raise ScanError("layer file member could not be read") + check_bytes(fileobj.read(), needles, stats, "layer-file") + except tarfile.TarError as error: + raise ScanError("layer is not a readable tar archive") from error + + +def scan_metadata(data: bytes, needles: Iterable[bytes], stats: Stats) -> None: + stats.metadata += 1 + check_bytes(data, needles, stats, "metadata") + + +def scan_docker_archive(path: Path, needles: Iterable[bytes]) -> Stats: + stats = Stats() + tar, members = open_archive(path) + try: + if DOCKER_ARCHIVE_MANIFEST not in members: + raise ScanError("Docker archive manifest is missing") + manifest_bytes = read_named_member(tar, members, DOCKER_ARCHIVE_MANIFEST) + scan_metadata(manifest_bytes, needles, stats) + manifest = parse_json(manifest_bytes, "Docker archive manifest") + if not isinstance(manifest, list) or not manifest: + raise ScanError("Docker archive manifest has no images") + + for image in manifest: + if not isinstance(image, dict): + raise ScanError("Docker archive image entry is invalid") + config_name = image.get("Config") + layers = image.get("Layers") + if not isinstance(config_name, str) or not isinstance(layers, list) or not layers: + raise ScanError("Docker archive image entry is incomplete") + stats.images += 1 + config_bytes = read_named_member(tar, members, config_name) + scan_metadata(config_bytes, needles, stats) + parse_json(config_bytes, "Docker image config") + for layer_name in layers: + if not isinstance(layer_name, str): + raise ScanError("Docker archive layer reference is invalid") + layer_bytes = read_named_member(tar, members, layer_name) + scan_layer_bytes(layer_bytes, needles, stats) + finally: + tar.close() + + if stats.images == 0 or stats.layers == 0: + raise ScanError("Docker archive did not contain any complete images") + return stats + + +def blob_path_from_digest(digest: str) -> str: + if not digest.startswith("sha256:"): + raise ScanError("only sha256 OCI digests are supported") + hex_digest = digest.split(":", 1)[1] + if len(hex_digest) != 64 or any(ch not in "0123456789abcdef" for ch in hex_digest): + raise ScanError("OCI digest is malformed") + return f"blobs/sha256/{hex_digest}" + + +def verify_digest(data: bytes, digest: str) -> None: + expected = digest.split(":", 1)[1] + actual = hashlib.sha256(data).hexdigest() + if actual != expected: + raise ScanError("OCI blob digest mismatch") + + +def read_oci_blob( + tar: tarfile.TarFile, + members: dict[str, tarfile.TarInfo], + descriptor: dict[str, Any], +) -> bytes: + digest = descriptor.get("digest") + if not isinstance(digest, str): + raise ScanError("OCI descriptor is missing a digest") + data = read_named_member(tar, members, blob_path_from_digest(digest)) + verify_digest(data, digest) + expected_size = descriptor.get("size") + if expected_size is not None and expected_size != len(data): + raise ScanError("OCI descriptor size mismatch") + return data + + +def scan_oci_descriptor( + tar: tarfile.TarFile, + members: dict[str, tarfile.TarInfo], + descriptor: dict[str, Any], + needles: Iterable[bytes], + stats: Stats, + visited: set[str], +) -> None: + media_type = descriptor.get("mediaType") + if not isinstance(media_type, str): + raise ScanError("OCI descriptor mediaType is missing") + data = read_oci_blob(tar, members, descriptor) + stats.blobs += 1 + + digest = descriptor["digest"] + if digest in visited: + return + visited.add(digest) + + if media_type in INDEX_MEDIA_TYPES: + scan_metadata(data, needles, stats) + index = parse_json(data, "OCI image index") + descriptors = index.get("manifests") + if not isinstance(descriptors, list) or not descriptors: + raise ScanError("OCI index contains no manifests") + for child in descriptors: + if not isinstance(child, dict): + raise ScanError("OCI index child descriptor is invalid") + scan_oci_descriptor(tar, members, child, needles, stats, visited) + return + + if media_type in MANIFEST_MEDIA_TYPES: + scan_metadata(data, needles, stats) + manifest = parse_json(data, "OCI image manifest") + config_desc = manifest.get("config") + layer_descs = manifest.get("layers") + if not isinstance(config_desc, dict) or not isinstance(layer_descs, list): + raise ScanError("OCI manifest is incomplete") + + config_type = config_desc.get("mediaType") + config_data = read_oci_blob(tar, members, config_desc) + stats.blobs += 1 + if isinstance(config_type, str) and (config_type in CONFIG_MEDIA_TYPES or config_type.endswith("+json")): + stats.images += 1 if config_type in CONFIG_MEDIA_TYPES else 0 + scan_metadata(config_data, needles, stats) + else: + raise ScanError("OCI config mediaType is unsupported") + + for layer_desc in layer_descs: + if not isinstance(layer_desc, dict): + raise ScanError("OCI layer descriptor is invalid") + layer_type = layer_desc.get("mediaType") + if not isinstance(layer_type, str): + raise ScanError("OCI layer mediaType is missing") + layer_data = read_oci_blob(tar, members, layer_desc) + stats.blobs += 1 + if layer_type in TAR_LAYER_MEDIA_TYPES or ".tar" in layer_type: + scan_layer_bytes(layer_data, needles, stats, layer_type) + elif layer_type in JSON_BLOB_MEDIA_TYPES or layer_type.endswith("+json"): + scan_metadata(layer_data, needles, stats) + else: + raise ScanError("OCI layer mediaType is unsupported") + return + + if media_type in CONFIG_MEDIA_TYPES or media_type in JSON_BLOB_MEDIA_TYPES or media_type.endswith("+json"): + scan_metadata(data, needles, stats) + return + + raise ScanError("OCI descriptor mediaType is unsupported") + + +def scan_oci_archive(path: Path, needles: Iterable[bytes]) -> Stats: + stats = Stats() + tar, members = open_archive(path) + try: + if OCI_LAYOUT not in members or OCI_INDEX not in members: + raise ScanError("OCI layout markers are missing") + layout_bytes = read_named_member(tar, members, OCI_LAYOUT) + scan_metadata(layout_bytes, needles, stats) + index_bytes = read_named_member(tar, members, OCI_INDEX) + scan_metadata(index_bytes, needles, stats) + index = parse_json(index_bytes, "OCI root index") + descriptors = index.get("manifests") + if not isinstance(descriptors, list) or not descriptors: + raise ScanError("OCI root index contains no manifests") + visited: set[str] = set() + for descriptor in descriptors: + if not isinstance(descriptor, dict): + raise ScanError("OCI root descriptor is invalid") + scan_oci_descriptor(tar, members, descriptor, needles, stats, visited) + finally: + tar.close() + + if stats.images == 0 or stats.layers == 0: + raise ScanError("OCI archive did not contain any complete image layers") + return stats + + +def scan_auto_archive(path: Path, needles: Iterable[bytes]) -> Stats: + tar, members = open_archive(path) + try: + member_names = set(members) + finally: + tar.close() + if OCI_LAYOUT in member_names and OCI_INDEX in member_names: + return scan_oci_archive(path, needles) + if DOCKER_ARCHIVE_MANIFEST in member_names: + return scan_docker_archive(path, needles) + raise ScanError("archive is neither Docker save nor OCI layout format") + + +def scan_docker_image(image: str, needles: Iterable[bytes]) -> Stats: + if shutil.which("docker") is None: + raise ScanError("docker CLI is not available") + with tempfile.TemporaryDirectory(prefix="image-secret-scan.") as tmp: + archive = Path(tmp) / "image.tar" + try: + subprocess.run( + ["docker", "image", "save", "--output", str(archive), image], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + except subprocess.CalledProcessError as error: + raise ScanError("docker image could not be exported for scanning") from error + return scan_docker_archive(archive, needles) + + +def add_blob(members: dict[str, bytes], payload: bytes) -> dict[str, Any]: + digest = hashlib.sha256(payload).hexdigest() + members[f"blobs/sha256/{digest}"] = payload + return {"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": f"sha256:{digest}", "size": len(payload)} + + +def make_layer(files: dict[str, bytes]) -> bytes: + raw = io.BytesIO() + with tarfile.open(fileobj=raw, mode="w") as tar: + for name, payload in files.items(): + info = tarfile.TarInfo(name=name) + info.size = len(payload) + info.mode = 0o600 + tar.addfile(info, io.BytesIO(payload)) + return gzip.compress(raw.getvalue()) + + +def make_oci_fixture(path: Path, *, leak: bool, needle: bytes) -> None: + blobs: dict[str, bytes] = {} + + config = json.dumps( + { + "architecture": "amd64", + "os": "linux", + "config": {"Env": ["PATH=/usr/local/bin"]}, + "rootfs": {"type": "layers", "diff_ids": []}, + "history": [{"created_by": "fixture"}], + }, + separators=(",", ":"), + ).encode() + config_digest = hashlib.sha256(config).hexdigest() + blobs[f"blobs/sha256/{config_digest}"] = config + config_desc = { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": f"sha256:{config_digest}", + "size": len(config), + } + + first_files = {"app/ok.txt": b"clean"} + if leak: + first_files["app/.bunfig.toml"] = b"registry=https://example.invalid/\n" + needle + b"\n" + layer1 = make_layer(first_files) + layer1_digest = hashlib.sha256(layer1).hexdigest() + blobs[f"blobs/sha256/{layer1_digest}"] = layer1 + + layer2 = make_layer({"app/.wh..bunfig.toml": b""} if leak else {"app/other.txt": b"clean"}) + layer2_digest = hashlib.sha256(layer2).hexdigest() + blobs[f"blobs/sha256/{layer2_digest}"] = layer2 + + manifest = json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config_desc, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": f"sha256:{layer1_digest}", + "size": len(layer1), + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": f"sha256:{layer2_digest}", + "size": len(layer2), + }, + ], + }, + separators=(",", ":"), + ).encode() + manifest_digest = hashlib.sha256(manifest).hexdigest() + blobs[f"blobs/sha256/{manifest_digest}"] = manifest + + nested_index = json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": f"sha256:{manifest_digest}", + "size": len(manifest), + "platform": {"os": "linux", "architecture": "amd64"}, + } + ], + }, + separators=(",", ":"), + ).encode() + nested_digest = hashlib.sha256(nested_index).hexdigest() + blobs[f"blobs/sha256/{nested_digest}"] = nested_index + + root_index = json.dumps( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.index.v1+json", + "digest": f"sha256:{nested_digest}", + "size": len(nested_index), + } + ], + }, + separators=(",", ":"), + ).encode() + + with tarfile.open(path, mode="w") as tar: + for name, payload in {"oci-layout": b'{"imageLayoutVersion":"1.0.0"}', "index.json": root_index, **blobs}.items(): + info = tarfile.TarInfo(name=name) + info.size = len(payload) + info.mode = 0o600 + tar.addfile(info, io.BytesIO(payload)) + + +def run_self_test() -> None: + needle = b"SCANNER_DELETED_LAYER_CANARY_20260909" + with tempfile.TemporaryDirectory(prefix="image-secret-scan-self-test.") as tmp: + tmp_path = Path(tmp) + needle_file = tmp_path / "needles.json" + needle_file.write_text( + json.dumps( + { + "schemaVersion": 1, + "needles": [base64.b64encode(needle).decode("ascii")], + } + ), + encoding="utf-8", + ) + needles = load_needles(needle_file) + + clean = tmp_path / "clean.oci.tar" + make_oci_fixture(clean, leak=False, needle=needle) + clean_stats = scan_oci_archive(clean, needles) + if clean_stats.hit_count: + raise ScanError("self-test clean fixture produced a match") + + leaky = tmp_path / "leaky.oci.tar" + make_oci_fixture(leaky, leak=True, needle=needle) + leaky_stats = scan_oci_archive(leaky, needles) + if leaky_stats.hit_count == 0: + raise ScanError("self-test did not catch a deleted lower-layer leak") + + malformed = tmp_path / "malformed.oci.tar" + with tarfile.open(malformed, mode="w") as tar: + payload = b'{"schemaVersion":2,"manifests":[]}' + for name, data in {"oci-layout": b'{"imageLayoutVersion":"1.0.0"}', "index.json": payload}.items(): + info = tarfile.TarInfo(name=name) + info.size = len(data) + tar.addfile(info, io.BytesIO(data)) + try: + scan_oci_archive(malformed, needles) + except ScanError: + pass + else: + raise ScanError("self-test malformed OCI layout did not fail closed") + + print("image secret scanner self-test passed") + + +def print_success(stats: Stats) -> None: + print( + "image secret scan passed: " + f"images={stats.images} layers={stats.layers} paths={stats.paths} " + f"metadata={stats.metadata}" + ) + + +def print_failure(stats: Stats) -> None: + categories = ",".join(f"{key}:{stats.hits[key]}" for key in sorted(stats.hits)) + print( + "error: forbidden Docker image material detected; " + f"hitCount={stats.hit_count}; categories={categories}; diagnostics=redacted", + file=sys.stderr, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Scan Docker/OCI images for private install config leaks") + parser.add_argument("--needles-file", type=Path, help="private JSON needle file created in runner temp") + parser.add_argument("--docker-image", help="local Docker image tag to export and scan") + parser.add_argument("--docker-archive", type=Path, help="docker save archive to scan") + parser.add_argument("--oci-archive", type=Path, help="OCI image-layout archive to scan") + parser.add_argument("--self-test", action="store_true", help="run scanner controls and exit") + args = parser.parse_args() + + try: + if args.self_test: + run_self_test() + return 0 + + targets = [args.docker_image, args.docker_archive, args.oci_archive] + if sum(value is not None for value in targets) != 1: + raise ScanError("choose exactly one image/archive target") + if args.needles_file is None: + raise ScanError("--needles-file is required") + needles = load_needles(args.needles_file) + + if args.docker_image: + stats = scan_docker_image(args.docker_image, needles) + elif args.docker_archive: + stats = scan_docker_archive(args.docker_archive, needles) + else: + stats = scan_oci_archive(args.oci_archive, needles) + + if stats.hit_count: + print_failure(stats) + return 1 + print_success(stats) + return 0 + except ScanError as error: + print(f"error: image secret scan failed closed: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From 0631dd60389386c51641de9a731f6975cd54e2d6 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Wed, 9 Sep 2026 10:51:41 -0400 Subject: [PATCH 5/7] Fix SFW image scan needle extraction --- .github/workflows/ci.yml | 12 +- .github/workflows/release.yml | 6 +- scripts/create-sfw-image-scan-needles.py | 398 ++++++++++++++++++++--- scripts/scan-image-secrets.py | 182 ++++++++--- 4 files changed, 515 insertions(+), 83 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bbda39f..6cecca3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,8 +143,10 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Test image secret scanner - run: python3 scripts/scan-image-secrets.py --self-test + - name: Test image scan tools + run: | + python3 scripts/create-sfw-image-scan-needles.py --self-test + python3 scripts/scan-image-secrets.py --self-test - name: Create private image scan needles if: steps.sfw.outputs.active == 'true' @@ -247,8 +249,10 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Test image secret scanner - run: python3 scripts/scan-image-secrets.py --self-test + - name: Test image scan tools + run: | + python3 scripts/create-sfw-image-scan-needles.py --self-test + python3 scripts/scan-image-secrets.py --self-test - name: Create private image scan needles if: steps.sfw.outputs.active == 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b4fcf30..b317798 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -316,8 +316,10 @@ jobs: echo "::error::Socket Firewall must be active before container publication." exit 1 - - name: Test image secret scanner - run: python3 scripts/scan-image-secrets.py --self-test + - name: Test image scan tools + run: | + python3 scripts/create-sfw-image-scan-needles.py --self-test + python3 scripts/scan-image-secrets.py --self-test - name: Create private image scan needles env: diff --git a/scripts/create-sfw-image-scan-needles.py b/scripts/create-sfw-image-scan-needles.py index c33c580..2d537ed 100755 --- a/scripts/create-sfw-image-scan-needles.py +++ b/scripts/create-sfw-image-scan-needles.py @@ -8,19 +8,25 @@ from __future__ import annotations +import argparse import base64 import json import os -import re import stat import sys +import tempfile from pathlib import Path +from urllib.parse import urlparse MIN_NEEDLE_LENGTH = 8 -TOKEN_RE = re.compile(rb"[A-Za-z0-9][A-Za-z0-9+/_:.,=@%?&#$~-]{7,}") -SENSITIVE_KEY_RE = re.compile( - rb"(?i)(auth|token|password|credential|secret|registry|socket|npm|bun)" -) +BUNFIG_BEGIN = "# >>> workos-sfw >>>" +BUNFIG_END = "# <<< workos-sfw <<<" +EXPECTED_SOCKET_REGISTRY = "https://socket-firewall.workos.dev/" +EXPECTED_SOCKET_HOST = "socket-firewall.workos.dev" + + +class ConfigError(RuntimeError): + """Raised when the active Socket Firewall Bun config is unsupported.""" def fail(message: str) -> None: @@ -37,52 +43,251 @@ def add_needle(needles: set[bytes], value: bytes) -> None: needles.add(value) -def derive_needles(config_bytes: bytes) -> list[bytes]: - needles: set[bytes] = set() - stripped_config = config_bytes.strip() - add_needle(needles, stripped_config) +def strip_toml_comment(line: str) -> str: + quote: str | None = None + escaped = False + for index, char in enumerate(line): + if quote == '"': + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + quote = None + elif char in "\r\n": + raise ConfigError("Socket Firewall Bun config is malformed") + continue + if quote == "'": + if char == "'": + quote = None + elif char in "\r\n": + raise ConfigError("Socket Firewall Bun config is malformed") + continue + if char == "#": + return line[:index] + if char in ("'", '"'): + quote = char + if quote is not None or escaped: + raise ConfigError("Socket Firewall Bun config is malformed") + return line + + +def extract_managed_block(config_bytes: bytes) -> str: + try: + text = config_bytes.decode("utf-8") + except UnicodeDecodeError as error: + raise ConfigError("Socket Firewall Bun config is not valid UTF-8") from error + if "\x00" in text: + raise ConfigError("Socket Firewall Bun config is malformed") - for raw_line in config_bytes.splitlines(): - line = raw_line.strip() - if not line or line.startswith(b"#"): + block_lines: list[str] = [] + inside = False + seen_block = False + for raw_line in text.splitlines(): + stripped = raw_line.strip() + if stripped == BUNFIG_BEGIN: + if inside or seen_block: + raise ConfigError("Socket Firewall Bun config has an unsupported managed block") + inside = True + seen_block = True continue + if stripped == BUNFIG_END: + if not inside: + raise ConfigError("Socket Firewall Bun config has an unsupported managed block") + inside = False + continue + if inside: + block_lines.append(raw_line) + continue + if stripped and not stripped.startswith("#"): + raise ConfigError("Socket Firewall Bun config contains unsupported active content") - if SENSITIVE_KEY_RE.search(line): - add_needle(needles, line) - # Capture common key/value forms without needing to know the exact - # config format emitted by the action. - for separator in (b"=", b":"): - if separator in line: - add_needle(needles, line.split(separator, 1)[1].strip().strip(b"'\"")) + if inside or not seen_block: + raise ConfigError("Socket Firewall Bun config managed block is missing") + return "\n".join(block_lines) - for token in TOKEN_RE.findall(line): - if b"socket" in token.lower() or SENSITIVE_KEY_RE.search(line): - add_needle(needles, token.strip(b"'\"")) - return sorted(needles, key=lambda item: (len(item), item)) +def parse_bare_key(text: str, index: int) -> tuple[str, int]: + start = index + while index < len(text) and (text[index].isalnum() or text[index] in "_-"): + index += 1 + if index == start: + raise ConfigError("Socket Firewall Bun config is malformed") + return text[start:index], index + + +def skip_space(text: str, index: int) -> int: + while index < len(text) and text[index] in " \t": + index += 1 + return index + + +def expect_char(text: str, index: int, expected: str) -> int: + index = skip_space(text, index) + if index >= len(text) or text[index] != expected: + raise ConfigError("Socket Firewall Bun config is malformed") + return index + 1 + + +def parse_toml_string(text: str, index: int) -> tuple[str, int]: + index = skip_space(text, index) + if index >= len(text) or text[index] not in ('"', "'"): + raise ConfigError("Socket Firewall Bun config is malformed") + quote = text[index] + start = index + index += 1 + escaped = False + while index < len(text): + char = text[index] + if quote == '"': + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif char == '"': + raw = text[start : index + 1] + try: + value = json.loads(raw) + except Exception as error: # noqa: BLE001 - redact config values + raise ConfigError("Socket Firewall Bun config is malformed") from error + if not isinstance(value, str): + raise ConfigError("Socket Firewall Bun config is malformed") + return value, index + 1 + elif char in "\r\n": + raise ConfigError("Socket Firewall Bun config is malformed") + else: + if char == "'": + return text[start + 1 : index], index + 1 + if char in "\r\n": + raise ConfigError("Socket Firewall Bun config is malformed") + index += 1 + raise ConfigError("Socket Firewall Bun config is malformed") -def main() -> int: - config_path = os.environ.get("SFW_BUN_CONFIG_FILE") - output_path = os.environ.get("SFW_IMAGE_SCAN_NEEDLES") - if not config_path: - fail("SFW_BUN_CONFIG_FILE is required") - if not output_path: - fail("SFW_IMAGE_SCAN_NEEDLES is required") - config = Path(config_path) - output = Path(output_path) +def parse_registry_inline_table(text: str, index: int) -> tuple[dict[str, str], int]: + values: dict[str, str] = {} + index = expect_char(text, index, "{") + while True: + index = skip_space(text, index) + if index >= len(text): + raise ConfigError("Socket Firewall Bun config is malformed") + if text[index] == "}": + return values, index + 1 + + key, index = parse_bare_key(text, index) + if key in values: + raise ConfigError("Socket Firewall Bun config has duplicate registry fields") + index = expect_char(text, index, "=") + value, index = parse_toml_string(text, index) + values[key] = value + + index = skip_space(text, index) + if index >= len(text): + raise ConfigError("Socket Firewall Bun config is malformed") + if text[index] == ",": + index += 1 + continue + if text[index] == "}": + return values, index + 1 + raise ConfigError("Socket Firewall Bun config is malformed") + + +def parse_registry_assignment(line: str) -> dict[str, str]: + index = skip_space(line, 0) + key, index = parse_bare_key(line, index) + if key != "registry": + raise ConfigError("Socket Firewall Bun config is malformed") + index = expect_char(line, index, "=") + values, index = parse_registry_inline_table(line, index) + if strip_toml_comment(line[index:]).strip(): + raise ConfigError("Socket Firewall Bun config is malformed") + if set(values) != {"url", "token"}: + raise ConfigError("Socket Firewall Bun config is missing required registry auth") + return values + + +def validate_toml_parser_agreement(managed_block: str, registry_url: str, token: str) -> None: try: - config_bytes = config.read_bytes() - except FileNotFoundError: - fail("Socket Firewall config was not created") + import tomllib # type: ignore[import-not-found] + except ModuleNotFoundError: + return - if not config_bytes.strip(): - fail("Socket Firewall config is empty") + try: + parsed = tomllib.loads(managed_block) + except Exception as error: # noqa: BLE001 - redact config values + raise ConfigError("Socket Firewall Bun config is malformed") from error - needles = derive_needles(config_bytes) + expected = {"install": {"registry": {"url": registry_url, "token": token}}} + if parsed != expected: + raise ConfigError("Socket Firewall Bun config shape is unsupported") + + +def parse_canonical_bun_config(config_bytes: bytes) -> tuple[str, str, bytes]: + managed_block = extract_managed_block(config_bytes) + active_lines: list[str] = [] + for raw_line in managed_block.splitlines(): + line = strip_toml_comment(raw_line).strip() + if line: + active_lines.append(line) + + if len(active_lines) != 2 or active_lines[0] != "[install]": + raise ConfigError("Socket Firewall Bun config shape is unsupported") + + registry_values = parse_registry_assignment(active_lines[1]) + registry_url = registry_values["url"] + token = registry_values["token"] + validate_toml_parser_agreement(managed_block, registry_url, token) + validate_registry_values(registry_url, token) + return registry_url, token, managed_block.strip().encode("utf-8") + + +def validate_registry_values(registry_url: str, token: str) -> None: + parsed = urlparse(registry_url) + if ( + registry_url != EXPECTED_SOCKET_REGISTRY + or parsed.scheme != "https" + or parsed.hostname != EXPECTED_SOCKET_HOST + or parsed.path != "/" + or parsed.params + or parsed.query + or parsed.fragment + ): + raise ConfigError("Socket Firewall registry URL is unsupported") + + token_bytes = token.encode("utf-8") + if len(token_bytes) < MIN_NEEDLE_LENGTH or token != token.strip(): + raise ConfigError("Socket Firewall registry auth token is unsupported") + if any(ord(char) < 0x20 or char == "\x7f" for char in token): + raise ConfigError("Socket Firewall registry auth token is unsupported") + if '"' in token or "\\" in token: + raise ConfigError("Socket Firewall registry auth token is unsupported") + + +def derive_needles(config_bytes: bytes) -> list[bytes]: + registry_url, token, managed_block = parse_canonical_bun_config(config_bytes) + token_bytes = token.encode("utf-8") + url_bytes = registry_url.encode("utf-8") + host_bytes = EXPECTED_SOCKET_HOST.encode("ascii") + + needles: set[bytes] = set() + add_needle(needles, token_bytes) + add_needle(needles, url_bytes) + add_needle(needles, host_bytes) + add_needle(needles, managed_block) + add_needle(needles, f'url = "{registry_url}"'.encode("utf-8")) + add_needle(needles, f'token = "{token}"'.encode("utf-8")) + add_needle(needles, f'//{EXPECTED_SOCKET_HOST}/:_authToken={token}'.encode("utf-8")) + add_needle(needles, f"BUN_CONFIG_REGISTRY={registry_url}".encode("utf-8")) + add_needle(needles, f"NPM_CONFIG_REGISTRY={registry_url}".encode("utf-8")) + add_needle(needles, f"PNPM_CONFIG_REGISTRY={registry_url}".encode("utf-8")) + + return sorted(needles, key=lambda item: (len(item), item)) + + +def write_needle_file(output: Path, needles: list[bytes]) -> None: if not needles: - fail("no scan needles could be derived from the Socket Firewall config") + raise ConfigError("no scan needles could be derived from the Socket Firewall config") payload = { "schemaVersion": 1, @@ -96,6 +301,121 @@ def main() -> int: handle.write("\n") os.chmod(output, stat.S_IRUSR | stat.S_IWUSR) + +def create_needle_file(config: Path, output: Path) -> list[bytes]: + try: + config_bytes = config.read_bytes() + except FileNotFoundError as error: + raise ConfigError("Socket Firewall config was not created") from error + + if not config_bytes.strip(): + raise ConfigError("Socket Firewall config is empty") + config_identity = config.resolve() + output_identity = output.resolve() if output.exists() else output.absolute() + if config_identity == output_identity: + raise ConfigError("Socket Firewall scan needle output path is unsupported") + + needles = derive_needles(config_bytes) + write_needle_file(output, needles) + return needles + + +def canonical_config(token: str) -> bytes: + return ( + f"\n{BUNFIG_BEGIN}\n" + "[install]\n" + f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}", token = "{token}" }}\n' + f"{BUNFIG_END}\n" + ).encode("utf-8") + + +def decode_payload_needles(path: Path) -> set[bytes]: + payload = json.loads(path.read_text(encoding="utf-8")) + return {base64.b64decode(item, validate=True) for item in payload["needles"]} + + +def run_self_test() -> None: + token = "SYNTHETIC_SOCKET_TOKEN_GENERATOR_20260909" + with tempfile.TemporaryDirectory(prefix="sfw-needle-generator-self-test.") as tmp: + tmp_path = Path(tmp) + config = tmp_path / ".bunfig.toml" + output = tmp_path / "needles.json" + config.write_bytes(canonical_config(token)) + needles = create_needle_file(config, output) + decoded = decode_payload_needles(output) + + required = { + token.encode("utf-8"), + EXPECTED_SOCKET_REGISTRY.encode("utf-8"), + EXPECTED_SOCKET_HOST.encode("ascii"), + } + if not required.issubset(decoded) or set(needles) != decoded: + raise ConfigError("self-test did not preserve required private needles") + if b"registry" in decoded or b"url" in decoded or b"token" in decoded: + raise ConfigError("self-test produced an unsafe bare TOML key needle") + if stat.S_IMODE(output.stat().st_mode) != 0o600: + raise ConfigError("self-test needle file permissions are not private") + + bad_config = tmp_path / "bad.bunfig.toml" + bad_output = tmp_path / "bad-needles.json" + bad_config.write_bytes( + ( + f"{BUNFIG_BEGIN}\n" + "[install]\n" + f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}" }}\n' + f"{BUNFIG_END}\n" + ).encode("utf-8") + ) + try: + create_needle_file(bad_config, bad_output) + except ConfigError: + pass + else: + raise ConfigError("self-test unsupported active config did not fail closed") + if bad_output.exists(): + raise ConfigError("self-test failure left a private needle file behind") + + unsupported = tmp_path / "unsupported.bunfig.toml" + unsupported.write_bytes( + ( + f"{BUNFIG_BEGIN}\n" + "[install]\n" + f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}", token = "{token}" }}\n' + "extra = true\n" + f"{BUNFIG_END}\n" + ).encode("utf-8") + ) + try: + derive_needles(unsupported.read_bytes()) + except ConfigError: + pass + else: + raise ConfigError("self-test extra active config did not fail closed") + + print("SFW image scan needle generator self-test passed") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Create private Docker image scan needles") + parser.add_argument("--self-test", action="store_true", help="run generator controls and exit") + args = parser.parse_args(argv) + + try: + if args.self_test: + run_self_test() + return 0 + + config_path = os.environ.get("SFW_BUN_CONFIG_FILE") + output_path = os.environ.get("SFW_IMAGE_SCAN_NEEDLES") + if not config_path: + raise ConfigError("SFW_BUN_CONFIG_FILE is required") + if not output_path: + raise ConfigError("SFW_IMAGE_SCAN_NEEDLES is required") + + needles = create_needle_file(Path(config_path), Path(output_path)) + except ConfigError as error: + fail(str(error)) + print(f"created private Docker image scan needles: count={len(needles)}") return 0 diff --git a/scripts/scan-image-secrets.py b/scripts/scan-image-secrets.py index f604ce2..0f993a9 100755 --- a/scripts/scan-image-secrets.py +++ b/scripts/scan-image-secrets.py @@ -464,7 +464,12 @@ def make_layer(files: dict[str, bytes]) -> bytes: return gzip.compress(raw.getvalue()) -def make_oci_fixture(path: Path, *, leak: bool, needle: bytes) -> None: +def make_oci_fixture( + path: Path, + *, + first_files: dict[str, bytes] | None = None, + second_files: dict[str, bytes] | None = None, +) -> None: blobs: dict[str, bytes] = {} config = json.dumps( @@ -485,34 +490,26 @@ def make_oci_fixture(path: Path, *, leak: bool, needle: bytes) -> None: "size": len(config), } - first_files = {"app/ok.txt": b"clean"} - if leak: - first_files["app/.bunfig.toml"] = b"registry=https://example.invalid/\n" + needle + b"\n" - layer1 = make_layer(first_files) - layer1_digest = hashlib.sha256(layer1).hexdigest() - blobs[f"blobs/sha256/{layer1_digest}"] = layer1 + def add_layer(files: dict[str, bytes]) -> dict[str, Any]: + layer = make_layer(files) + layer_digest = hashlib.sha256(layer).hexdigest() + blobs[f"blobs/sha256/{layer_digest}"] = layer + return { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": f"sha256:{layer_digest}", + "size": len(layer), + } - layer2 = make_layer({"app/.wh..bunfig.toml": b""} if leak else {"app/other.txt": b"clean"}) - layer2_digest = hashlib.sha256(layer2).hexdigest() - blobs[f"blobs/sha256/{layer2_digest}"] = layer2 + layer_descs = [add_layer(first_files or {"app/ok.txt": b"clean"})] + if second_files is not None: + layer_descs.append(add_layer(second_files)) manifest = json.dumps( { "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", "config": config_desc, - "layers": [ - { - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": f"sha256:{layer1_digest}", - "size": len(layer1), - }, - { - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": f"sha256:{layer2_digest}", - "size": len(layer2), - }, - ], + "layers": layer_descs, }, separators=(",", ":"), ).encode() @@ -560,33 +557,133 @@ def make_oci_fixture(path: Path, *, leak: bool, needle: bytes) -> None: tar.addfile(info, io.BytesIO(payload)) +def run_generator_for_self_test(config: Path, output: Path) -> None: + generator = Path(__file__).with_name("create-sfw-image-scan-needles.py") + env = { + **os.environ, + "SFW_BUN_CONFIG_FILE": str(config), + "SFW_IMAGE_SCAN_NEEDLES": str(output), + } + result = subprocess.run( + [sys.executable, str(generator)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + if result.returncode != 0: + raise ScanError("self-test needle generator failed") + + +def expect_generator_failure(config: Path, output: Path, protected_values: Iterable[str]) -> None: + generator = Path(__file__).with_name("create-sfw-image-scan-needles.py") + env = { + **os.environ, + "SFW_BUN_CONFIG_FILE": str(config), + "SFW_IMAGE_SCAN_NEEDLES": str(output), + } + result = subprocess.run( + [sys.executable, str(generator)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + if result.returncode == 0: + raise ScanError("self-test unsupported generator config did not fail closed") + if output.exists(): + raise ScanError("self-test generator failure left a needle file behind") + diagnostics = result.stdout + result.stderr + if any(value and value in diagnostics for value in protected_values): + raise ScanError("self-test generator diagnostics disclosed a protected value") + + +def expect_cli_scan_failure( + archive: Path, + needle_file: Path, + protected_values: Iterable[str], +) -> None: + result = subprocess.run( + [sys.executable, __file__, "--oci-archive", str(archive), "--needles-file", str(needle_file)], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 1: + raise ScanError("self-test leaky fixture did not fail with forbidden material") + diagnostics = result.stdout + result.stderr + if any(value and value in diagnostics for value in protected_values): + raise ScanError("self-test scanner diagnostics disclosed a protected value") + + def run_self_test() -> None: - needle = b"SCANNER_DELETED_LAYER_CANARY_20260909" + token = "SYNTHETIC_SOCKET_TOKEN_SCANNER_20260909" + endpoint = "https://socket-firewall.workos.dev/" + host = "socket-firewall.workos.dev" + begin = "# >>> workos-sfw >>>" + end = "# <<< workos-sfw <<<" with tempfile.TemporaryDirectory(prefix="image-secret-scan-self-test.") as tmp: tmp_path = Path(tmp) - needle_file = tmp_path / "needles.json" - needle_file.write_text( - json.dumps( - { - "schemaVersion": 1, - "needles": [base64.b64encode(needle).decode("ascii")], - } - ), + config = tmp_path / ".bunfig.toml" + config.write_text( + f"\n{begin}\n[install]\nregistry = {{ url = \"{endpoint}\", token = \"{token}\" }}\n{end}\n", encoding="utf-8", ) + needle_file = tmp_path / "needles.json" + run_generator_for_self_test(config, needle_file) needles = load_needles(needle_file) + if b"registry" in needles or b"url" in needles or b"token" in needles: + raise ScanError("self-test generator produced a bare TOML key needle") + for required in (token.encode(), endpoint.encode(), host.encode()): + if required not in needles: + raise ScanError("self-test generator did not preserve required scan values") clean = tmp_path / "clean.oci.tar" - make_oci_fixture(clean, leak=False, needle=needle) + make_oci_fixture( + clean, + first_files={ + "app/registry-map.js": b"const registry = new Map();\n", + "app/node_modules/npm-registry-fetch/index.js": b"module.exports = 'registry';\n", + }, + second_files={"app/other.txt": b"clean"}, + ) clean_stats = scan_oci_archive(clean, needles) if clean_stats.hit_count: - raise ScanError("self-test clean fixture produced a match") - - leaky = tmp_path / "leaky.oci.tar" - make_oci_fixture(leaky, leak=True, needle=needle) - leaky_stats = scan_oci_archive(leaky, needles) - if leaky_stats.hit_count == 0: + raise ScanError("self-test clean registry-word fixture produced a match") + + credential = tmp_path / "credential.oci.tar" + make_oci_fixture(credential, first_files={"app/token.txt": token.encode()}) + credential_stats = scan_oci_archive(credential, needles) + if credential_stats.hit_count == 0: + raise ScanError("self-test did not catch a credential leak") + + endpoint_leak = tmp_path / "endpoint.oci.tar" + make_oci_fixture(endpoint_leak, first_files={"app/endpoint.txt": endpoint.encode()}) + endpoint_stats = scan_oci_archive(endpoint_leak, needles) + if endpoint_stats.hit_count == 0: + raise ScanError("self-test did not catch a Socket endpoint leak") + + deleted_lower_layer = tmp_path / "deleted-lower-layer.oci.tar" + make_oci_fixture( + deleted_lower_layer, + first_files={ + "app/.bunfig.toml": ( + f'registry = {{ url = "{endpoint}", token = "{token}" }}\n' + ).encode(), + }, + second_files={"app/.wh..bunfig.toml": b""}, + ) + deleted_stats = scan_oci_archive(deleted_lower_layer, needles) + if deleted_stats.hit_count == 0: raise ScanError("self-test did not catch a deleted lower-layer leak") + expect_cli_scan_failure( + deleted_lower_layer, + needle_file, + [token, endpoint, host, "app/.bunfig.toml", str(needle_file), str(config)], + ) malformed = tmp_path / "malformed.oci.tar" with tarfile.open(malformed, mode="w") as tar: @@ -602,9 +699,18 @@ def run_self_test() -> None: else: raise ScanError("self-test malformed OCI layout did not fail closed") + bad_config = tmp_path / "bad.bunfig.toml" + bad_output = tmp_path / "bad-needles.json" + bad_config.write_text( + f"{begin}\n[install]\nregistry = {{ url = \"{endpoint}\", token = \"{token}\" }}\nextra = true\n{end}\n", + encoding="utf-8", + ) + expect_generator_failure(bad_config, bad_output, [token, endpoint, host, str(bad_config)]) + print("image secret scanner self-test passed") + def print_success(stats: Stats) -> None: print( "image secret scan passed: " From 878d21f27a72a9718ea4f8f64483ddb6d9248f48 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 10 Sep 2026 12:55:40 -0400 Subject: [PATCH 6/7] Remove recurring image scan gates --- .github/workflows/ci.yml | 88 +-- .github/workflows/release.yml | 45 +- scripts/create-sfw-image-scan-needles.py | 424 ------------- scripts/scan-image-secrets.py | 770 ----------------------- 4 files changed, 16 insertions(+), 1311 deletions(-) delete mode 100755 scripts/create-sfw-image-scan-needles.py delete mode 100755 scripts/scan-image-secrets.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6cecca3..1dfa045 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,19 +143,7 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Test image scan tools - run: | - python3 scripts/create-sfw-image-scan-needles.py --self-test - python3 scripts/scan-image-secrets.py --self-test - - - name: Create private image scan needles - if: steps.sfw.outputs.active == 'true' - env: - SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/create-sfw-image-scan-needles.py - - - name: Build authenticated image for scan + - name: Build image with Socket Firewall if: steps.sfw.outputs.active == 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: @@ -172,43 +160,21 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - - name: Scan authenticated image - if: steps.sfw.outputs.active == 'true' - env: - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/scan-image-secrets.py --docker-image emulate:ci --needles-file "$SFW_IMAGE_SCAN_NEEDLES" - - name: Teardown Socket Firewall if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - - name: Remove private Docker scan materials + - name: Remove private Docker install config if: ${{ always() }} env: SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json run: | - rm -f "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" - if [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then - echo "::error::Private Docker scan material was not removed." + rm -f "$SFW_BUN_CONFIG_FILE" + if [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker install config was not removed." exit 1 fi - - name: Load scanned image from clean cache (publication disabled) - if: steps.sfw.outputs.active == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - push: false - load: true - platforms: linux/amd64 - tags: emulate:ci - build-args: | - SFW_REQUIRED=true - add-hosts: | - registry.npmjs.org:127.0.0.1 - registry.yarnpkg.com:127.0.0.1 - - name: Build image (external fork fallback) if: steps.sfw.outputs.active == 'false' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 @@ -249,19 +215,7 @@ jobs: allow-external-fork-fallback: true configure-bun: true - - name: Test image scan tools - run: | - python3 scripts/create-sfw-image-scan-needles.py --self-test - python3 scripts/scan-image-secrets.py --self-test - - - name: Create private image scan needles - if: steps.sfw.outputs.active == 'true' - env: - SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/create-sfw-image-scan-needles.py - - - name: Build authenticated image for scan + - name: Build image with Socket Firewall if: steps.sfw.outputs.active == 'true' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: @@ -278,43 +232,21 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - - name: Scan authenticated image - if: steps.sfw.outputs.active == 'true' - env: - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/scan-image-secrets.py --docker-image emulate:ci --needles-file "$SFW_IMAGE_SCAN_NEEDLES" - - name: Teardown Socket Firewall if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - - name: Remove private Docker scan materials + - name: Remove private Docker install config if: ${{ always() }} env: SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json run: | - rm -f "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" - if [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then - echo "::error::Private Docker scan material was not removed." + rm -f "$SFW_BUN_CONFIG_FILE" + if [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker install config was not removed." exit 1 fi - - name: Load scanned image from clean cache (publication disabled) - if: steps.sfw.outputs.active == 'true' - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - push: false - load: true - platforms: linux/arm64 - tags: emulate:ci - build-args: | - SFW_REQUIRED=true - add-hosts: | - registry.npmjs.org:127.0.0.1 - registry.yarnpkg.com:127.0.0.1 - - name: Build image (external fork fallback) if: steps.sfw.outputs.active == 'false' uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b317798..90b89c1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -316,24 +316,12 @@ jobs: echo "::error::Socket Firewall must be active before container publication." exit 1 - - name: Test image scan tools - run: | - python3 scripts/create-sfw-image-scan-needles.py --self-test - python3 scripts/scan-image-secrets.py --self-test - - - name: Create private image scan needles - env: - SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/create-sfw-image-scan-needles.py - - - name: Build authenticated image archive for scan + - name: Publish image with Socket Firewall uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 with: context: . platforms: linux/amd64,linux/arm64 - push: false - outputs: type=oci,dest=${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar + push: true tags: ${{ steps.tags.outputs.tags }} build-args: | SFW_REQUIRED=true @@ -343,42 +331,21 @@ jobs: secret-files: | sfw_bunfig=${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - - name: Scan authenticated image archive - env: - EMULATE_IMAGE_ARCHIVE: ${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json - run: python3 scripts/scan-image-secrets.py --oci-archive "$EMULATE_IMAGE_ARCHIVE" --needles-file "$SFW_IMAGE_SCAN_NEEDLES" - - name: Teardown Socket Firewall if: ${{ always() && steps.sfw.outputs.active == 'true' }} uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d - - name: Remove private Docker scan materials + - name: Remove private Docker install config if: ${{ always() }} env: - EMULATE_IMAGE_ARCHIVE: ${{ runner.temp }}/emulate-ghcr-prepublish.oci.tar SFW_BUN_CONFIG_FILE: ${{ runner.temp }}/sfw-docker-bun/.bunfig.toml - SFW_IMAGE_SCAN_NEEDLES: ${{ runner.temp }}/sfw-image-scan-needles.json run: | - rm -f "$EMULATE_IMAGE_ARCHIVE" "$SFW_IMAGE_SCAN_NEEDLES" "$SFW_BUN_CONFIG_FILE" - if [ -e "$EMULATE_IMAGE_ARCHIVE" ] || [ -e "$SFW_IMAGE_SCAN_NEEDLES" ] || [ -e "$SFW_BUN_CONFIG_FILE" ]; then - echo "::error::Private Docker scan material was not removed." + rm -f "$SFW_BUN_CONFIG_FILE" + if [ -e "$SFW_BUN_CONFIG_FILE" ]; then + echo "::error::Private Docker install config was not removed." exit 1 fi - - name: Publish scanned image from clean cache - uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6.19.2 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.tags.outputs.tags }} - build-args: | - SFW_REQUIRED=true - add-hosts: | - registry.npmjs.org:127.0.0.1 - registry.yarnpkg.com:127.0.0.1 - homebrew: name: Update Homebrew formula runs-on: macos-15 diff --git a/scripts/create-sfw-image-scan-needles.py b/scripts/create-sfw-image-scan-needles.py deleted file mode 100755 index 2d537ed..0000000 --- a/scripts/create-sfw-image-scan-needles.py +++ /dev/null @@ -1,424 +0,0 @@ -#!/usr/bin/env python3 -"""Create a private needle file for Docker image leak scans. - -The input is the temporary Bun config produced by the pinned Socket Firewall -setup action. The output may contain credentials and must stay in runner temp. -This script intentionally prints only counts, never paths or values. -""" - -from __future__ import annotations - -import argparse -import base64 -import json -import os -import stat -import sys -import tempfile -from pathlib import Path -from urllib.parse import urlparse - -MIN_NEEDLE_LENGTH = 8 -BUNFIG_BEGIN = "# >>> workos-sfw >>>" -BUNFIG_END = "# <<< workos-sfw <<<" -EXPECTED_SOCKET_REGISTRY = "https://socket-firewall.workos.dev/" -EXPECTED_SOCKET_HOST = "socket-firewall.workos.dev" - - -class ConfigError(RuntimeError): - """Raised when the active Socket Firewall Bun config is unsupported.""" - - -def fail(message: str) -> None: - print(f"error: {message}", file=sys.stderr) - sys.exit(1) - - -def add_needle(needles: set[bytes], value: bytes) -> None: - value = value.strip() - if len(value) < MIN_NEEDLE_LENGTH: - return - if not any(byte > 0x20 for byte in value): - return - needles.add(value) - - -def strip_toml_comment(line: str) -> str: - quote: str | None = None - escaped = False - for index, char in enumerate(line): - if quote == '"': - if escaped: - escaped = False - elif char == "\\": - escaped = True - elif char == '"': - quote = None - elif char in "\r\n": - raise ConfigError("Socket Firewall Bun config is malformed") - continue - if quote == "'": - if char == "'": - quote = None - elif char in "\r\n": - raise ConfigError("Socket Firewall Bun config is malformed") - continue - if char == "#": - return line[:index] - if char in ("'", '"'): - quote = char - if quote is not None or escaped: - raise ConfigError("Socket Firewall Bun config is malformed") - return line - - -def extract_managed_block(config_bytes: bytes) -> str: - try: - text = config_bytes.decode("utf-8") - except UnicodeDecodeError as error: - raise ConfigError("Socket Firewall Bun config is not valid UTF-8") from error - if "\x00" in text: - raise ConfigError("Socket Firewall Bun config is malformed") - - block_lines: list[str] = [] - inside = False - seen_block = False - for raw_line in text.splitlines(): - stripped = raw_line.strip() - if stripped == BUNFIG_BEGIN: - if inside or seen_block: - raise ConfigError("Socket Firewall Bun config has an unsupported managed block") - inside = True - seen_block = True - continue - if stripped == BUNFIG_END: - if not inside: - raise ConfigError("Socket Firewall Bun config has an unsupported managed block") - inside = False - continue - if inside: - block_lines.append(raw_line) - continue - if stripped and not stripped.startswith("#"): - raise ConfigError("Socket Firewall Bun config contains unsupported active content") - - if inside or not seen_block: - raise ConfigError("Socket Firewall Bun config managed block is missing") - return "\n".join(block_lines) - - -def parse_bare_key(text: str, index: int) -> tuple[str, int]: - start = index - while index < len(text) and (text[index].isalnum() or text[index] in "_-"): - index += 1 - if index == start: - raise ConfigError("Socket Firewall Bun config is malformed") - return text[start:index], index - - -def skip_space(text: str, index: int) -> int: - while index < len(text) and text[index] in " \t": - index += 1 - return index - - -def expect_char(text: str, index: int, expected: str) -> int: - index = skip_space(text, index) - if index >= len(text) or text[index] != expected: - raise ConfigError("Socket Firewall Bun config is malformed") - return index + 1 - - -def parse_toml_string(text: str, index: int) -> tuple[str, int]: - index = skip_space(text, index) - if index >= len(text) or text[index] not in ('"', "'"): - raise ConfigError("Socket Firewall Bun config is malformed") - - quote = text[index] - start = index - index += 1 - escaped = False - while index < len(text): - char = text[index] - if quote == '"': - if escaped: - escaped = False - elif char == "\\": - escaped = True - elif char == '"': - raw = text[start : index + 1] - try: - value = json.loads(raw) - except Exception as error: # noqa: BLE001 - redact config values - raise ConfigError("Socket Firewall Bun config is malformed") from error - if not isinstance(value, str): - raise ConfigError("Socket Firewall Bun config is malformed") - return value, index + 1 - elif char in "\r\n": - raise ConfigError("Socket Firewall Bun config is malformed") - else: - if char == "'": - return text[start + 1 : index], index + 1 - if char in "\r\n": - raise ConfigError("Socket Firewall Bun config is malformed") - index += 1 - raise ConfigError("Socket Firewall Bun config is malformed") - - -def parse_registry_inline_table(text: str, index: int) -> tuple[dict[str, str], int]: - values: dict[str, str] = {} - index = expect_char(text, index, "{") - while True: - index = skip_space(text, index) - if index >= len(text): - raise ConfigError("Socket Firewall Bun config is malformed") - if text[index] == "}": - return values, index + 1 - - key, index = parse_bare_key(text, index) - if key in values: - raise ConfigError("Socket Firewall Bun config has duplicate registry fields") - index = expect_char(text, index, "=") - value, index = parse_toml_string(text, index) - values[key] = value - - index = skip_space(text, index) - if index >= len(text): - raise ConfigError("Socket Firewall Bun config is malformed") - if text[index] == ",": - index += 1 - continue - if text[index] == "}": - return values, index + 1 - raise ConfigError("Socket Firewall Bun config is malformed") - - -def parse_registry_assignment(line: str) -> dict[str, str]: - index = skip_space(line, 0) - key, index = parse_bare_key(line, index) - if key != "registry": - raise ConfigError("Socket Firewall Bun config is malformed") - index = expect_char(line, index, "=") - values, index = parse_registry_inline_table(line, index) - if strip_toml_comment(line[index:]).strip(): - raise ConfigError("Socket Firewall Bun config is malformed") - if set(values) != {"url", "token"}: - raise ConfigError("Socket Firewall Bun config is missing required registry auth") - return values - - -def validate_toml_parser_agreement(managed_block: str, registry_url: str, token: str) -> None: - try: - import tomllib # type: ignore[import-not-found] - except ModuleNotFoundError: - return - - try: - parsed = tomllib.loads(managed_block) - except Exception as error: # noqa: BLE001 - redact config values - raise ConfigError("Socket Firewall Bun config is malformed") from error - - expected = {"install": {"registry": {"url": registry_url, "token": token}}} - if parsed != expected: - raise ConfigError("Socket Firewall Bun config shape is unsupported") - - -def parse_canonical_bun_config(config_bytes: bytes) -> tuple[str, str, bytes]: - managed_block = extract_managed_block(config_bytes) - active_lines: list[str] = [] - for raw_line in managed_block.splitlines(): - line = strip_toml_comment(raw_line).strip() - if line: - active_lines.append(line) - - if len(active_lines) != 2 or active_lines[0] != "[install]": - raise ConfigError("Socket Firewall Bun config shape is unsupported") - - registry_values = parse_registry_assignment(active_lines[1]) - registry_url = registry_values["url"] - token = registry_values["token"] - validate_toml_parser_agreement(managed_block, registry_url, token) - validate_registry_values(registry_url, token) - return registry_url, token, managed_block.strip().encode("utf-8") - - -def validate_registry_values(registry_url: str, token: str) -> None: - parsed = urlparse(registry_url) - if ( - registry_url != EXPECTED_SOCKET_REGISTRY - or parsed.scheme != "https" - or parsed.hostname != EXPECTED_SOCKET_HOST - or parsed.path != "/" - or parsed.params - or parsed.query - or parsed.fragment - ): - raise ConfigError("Socket Firewall registry URL is unsupported") - - token_bytes = token.encode("utf-8") - if len(token_bytes) < MIN_NEEDLE_LENGTH or token != token.strip(): - raise ConfigError("Socket Firewall registry auth token is unsupported") - if any(ord(char) < 0x20 or char == "\x7f" for char in token): - raise ConfigError("Socket Firewall registry auth token is unsupported") - if '"' in token or "\\" in token: - raise ConfigError("Socket Firewall registry auth token is unsupported") - - -def derive_needles(config_bytes: bytes) -> list[bytes]: - registry_url, token, managed_block = parse_canonical_bun_config(config_bytes) - token_bytes = token.encode("utf-8") - url_bytes = registry_url.encode("utf-8") - host_bytes = EXPECTED_SOCKET_HOST.encode("ascii") - - needles: set[bytes] = set() - add_needle(needles, token_bytes) - add_needle(needles, url_bytes) - add_needle(needles, host_bytes) - add_needle(needles, managed_block) - add_needle(needles, f'url = "{registry_url}"'.encode("utf-8")) - add_needle(needles, f'token = "{token}"'.encode("utf-8")) - add_needle(needles, f'//{EXPECTED_SOCKET_HOST}/:_authToken={token}'.encode("utf-8")) - add_needle(needles, f"BUN_CONFIG_REGISTRY={registry_url}".encode("utf-8")) - add_needle(needles, f"NPM_CONFIG_REGISTRY={registry_url}".encode("utf-8")) - add_needle(needles, f"PNPM_CONFIG_REGISTRY={registry_url}".encode("utf-8")) - - return sorted(needles, key=lambda item: (len(item), item)) - - -def write_needle_file(output: Path, needles: list[bytes]) -> None: - if not needles: - raise ConfigError("no scan needles could be derived from the Socket Firewall config") - - payload = { - "schemaVersion": 1, - "needles": [base64.b64encode(needle).decode("ascii") for needle in needles], - } - - output.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(output, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - with os.fdopen(fd, "w", encoding="utf-8") as handle: - json.dump(payload, handle, separators=(",", ":")) - handle.write("\n") - os.chmod(output, stat.S_IRUSR | stat.S_IWUSR) - - -def create_needle_file(config: Path, output: Path) -> list[bytes]: - try: - config_bytes = config.read_bytes() - except FileNotFoundError as error: - raise ConfigError("Socket Firewall config was not created") from error - - if not config_bytes.strip(): - raise ConfigError("Socket Firewall config is empty") - config_identity = config.resolve() - output_identity = output.resolve() if output.exists() else output.absolute() - if config_identity == output_identity: - raise ConfigError("Socket Firewall scan needle output path is unsupported") - - needles = derive_needles(config_bytes) - write_needle_file(output, needles) - return needles - - -def canonical_config(token: str) -> bytes: - return ( - f"\n{BUNFIG_BEGIN}\n" - "[install]\n" - f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}", token = "{token}" }}\n' - f"{BUNFIG_END}\n" - ).encode("utf-8") - - -def decode_payload_needles(path: Path) -> set[bytes]: - payload = json.loads(path.read_text(encoding="utf-8")) - return {base64.b64decode(item, validate=True) for item in payload["needles"]} - - -def run_self_test() -> None: - token = "SYNTHETIC_SOCKET_TOKEN_GENERATOR_20260909" - with tempfile.TemporaryDirectory(prefix="sfw-needle-generator-self-test.") as tmp: - tmp_path = Path(tmp) - config = tmp_path / ".bunfig.toml" - output = tmp_path / "needles.json" - config.write_bytes(canonical_config(token)) - needles = create_needle_file(config, output) - decoded = decode_payload_needles(output) - - required = { - token.encode("utf-8"), - EXPECTED_SOCKET_REGISTRY.encode("utf-8"), - EXPECTED_SOCKET_HOST.encode("ascii"), - } - if not required.issubset(decoded) or set(needles) != decoded: - raise ConfigError("self-test did not preserve required private needles") - if b"registry" in decoded or b"url" in decoded or b"token" in decoded: - raise ConfigError("self-test produced an unsafe bare TOML key needle") - if stat.S_IMODE(output.stat().st_mode) != 0o600: - raise ConfigError("self-test needle file permissions are not private") - - bad_config = tmp_path / "bad.bunfig.toml" - bad_output = tmp_path / "bad-needles.json" - bad_config.write_bytes( - ( - f"{BUNFIG_BEGIN}\n" - "[install]\n" - f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}" }}\n' - f"{BUNFIG_END}\n" - ).encode("utf-8") - ) - try: - create_needle_file(bad_config, bad_output) - except ConfigError: - pass - else: - raise ConfigError("self-test unsupported active config did not fail closed") - if bad_output.exists(): - raise ConfigError("self-test failure left a private needle file behind") - - unsupported = tmp_path / "unsupported.bunfig.toml" - unsupported.write_bytes( - ( - f"{BUNFIG_BEGIN}\n" - "[install]\n" - f'registry = {{ url = "{EXPECTED_SOCKET_REGISTRY}", token = "{token}" }}\n' - "extra = true\n" - f"{BUNFIG_END}\n" - ).encode("utf-8") - ) - try: - derive_needles(unsupported.read_bytes()) - except ConfigError: - pass - else: - raise ConfigError("self-test extra active config did not fail closed") - - print("SFW image scan needle generator self-test passed") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description="Create private Docker image scan needles") - parser.add_argument("--self-test", action="store_true", help="run generator controls and exit") - args = parser.parse_args(argv) - - try: - if args.self_test: - run_self_test() - return 0 - - config_path = os.environ.get("SFW_BUN_CONFIG_FILE") - output_path = os.environ.get("SFW_IMAGE_SCAN_NEEDLES") - if not config_path: - raise ConfigError("SFW_BUN_CONFIG_FILE is required") - if not output_path: - raise ConfigError("SFW_IMAGE_SCAN_NEEDLES is required") - - needles = create_needle_file(Path(config_path), Path(output_path)) - except ConfigError as error: - fail(str(error)) - - print(f"created private Docker image scan needles: count={len(needles)}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/scan-image-secrets.py b/scripts/scan-image-secrets.py deleted file mode 100755 index 0f993a9..0000000 --- a/scripts/scan-image-secrets.py +++ /dev/null @@ -1,770 +0,0 @@ -#!/usr/bin/env python3 -"""Fail-closed Docker/OCI image scanner for ephemeral install config leaks. - -The scanner inspects image config/history metadata and every layer tar member, -including deleted lower-layer files. Diagnostics intentionally redact matched -values, filenames, and archive member paths. -""" - -from __future__ import annotations - -import argparse -import base64 -import gzip -import hashlib -import io -import json -import os -import shutil -import subprocess -import sys -import tarfile -import tempfile -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Iterable - -DOCKER_ARCHIVE_MANIFEST = "manifest.json" -OCI_LAYOUT = "oci-layout" -OCI_INDEX = "index.json" - -INDEX_MEDIA_TYPES = { - "application/vnd.oci.image.index.v1+json", - "application/vnd.docker.distribution.manifest.list.v2+json", -} -MANIFEST_MEDIA_TYPES = { - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", -} -CONFIG_MEDIA_TYPES = { - "application/vnd.oci.image.config.v1+json", - "application/vnd.docker.container.image.v1+json", -} -TAR_LAYER_MEDIA_TYPES = { - "application/vnd.oci.image.layer.v1.tar", - "application/vnd.oci.image.layer.v1.tar+gzip", - "application/vnd.oci.image.layer.nondistributable.v1.tar", - "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", - "application/vnd.docker.image.rootfs.diff.tar", - "application/vnd.docker.image.rootfs.diff.tar.gzip", - "application/vnd.docker.image.rootfs.foreign.diff.tar", - "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", -} -JSON_BLOB_MEDIA_TYPES = { - "application/vnd.in-toto+json", - "application/vnd.dsse.envelope.v1+json", - "application/vnd.oci.empty.v1+json", -} - -# These catch committed package-manager config files or credential declarations -# even if the dynamic credential value itself is transformed. They are not -# printed when matched. -FORBIDDEN_PATH_NAMES = { - ".bunfig.toml", - "bunfig.toml", -} -FORBIDDEN_CONTENT_MARKERS = [ - b"PUBLIC_SOCKET_FIREWALL_TOKEN", - b"SOCKET_FIREWALL_TOKEN", -] - - -class ScanError(RuntimeError): - """Raised when an archive cannot be fully inspected.""" - - -@dataclass -class Stats: - images: int = 0 - layers: int = 0 - paths: int = 0 - metadata: int = 0 - blobs: int = 0 - hits: dict[str, int] = field(default_factory=dict) - - def hit(self, kind: str) -> None: - self.hits[kind] = self.hits.get(kind, 0) + 1 - - @property - def hit_count(self) -> int: - return sum(self.hits.values()) - - def merge(self, other: "Stats") -> None: - self.images += other.images - self.layers += other.layers - self.paths += other.paths - self.metadata += other.metadata - self.blobs += other.blobs - for kind, count in other.hits.items(): - self.hits[kind] = self.hits.get(kind, 0) + count - - -def load_needles(path: Path) -> list[bytes]: - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except Exception as error: # noqa: BLE001 - fail closed with concise cause - raise ScanError("needle file is not valid JSON") from error - - encoded = payload.get("needles") - if payload.get("schemaVersion") != 1 or not isinstance(encoded, list): - raise ScanError("needle file schema is invalid") - - needles: list[bytes] = [] - for item in encoded: - if not isinstance(item, str): - raise ScanError("needle file contains a non-string entry") - try: - value = base64.b64decode(item, validate=True) - except Exception as error: # noqa: BLE001 - raise ScanError("needle file contains invalid base64") from error - if len(value) >= 8: - needles.append(value) - - # Deduplicate while preserving deterministic order. - seen: set[bytes] = set() - unique = [] - for needle in needles: - if needle not in seen: - unique.append(needle) - seen.add(needle) - - if not unique: - raise ScanError("needle file contained no usable scan needles") - return unique - - -def check_bytes(data: bytes, needles: Iterable[bytes], stats: Stats, location: str) -> None: - del location # Locations are intentionally redacted from diagnostics. - lower_data = data.lower() - for needle in needles: - if needle and needle in data: - stats.hit("dynamic-needle") - for marker in FORBIDDEN_CONTENT_MARKERS: - if marker.lower() in lower_data: - stats.hit("credential-marker") - - -def check_path(name: str, needles: Iterable[bytes], stats: Stats) -> None: - encoded = name.encode("utf-8", "surrogateescape") - for part in name.replace("\\", "/").split("/"): - normalized = part[4:] if part.startswith(".wh.") else part - if normalized in FORBIDDEN_PATH_NAMES: - stats.hit("forbidden-package-config-path") - for needle in needles: - if needle and needle in encoded: - stats.hit("dynamic-needle-in-path") - - -def safe_tar_members(tar: tarfile.TarFile) -> dict[str, tarfile.TarInfo]: - members: dict[str, tarfile.TarInfo] = {} - for member in tar.getmembers(): - name = member.name - normalized = Path(name) - if normalized.is_absolute() or ".." in normalized.parts: - raise ScanError("archive contains an unsafe member path") - if name in members: - raise ScanError("archive contains duplicate member paths") - members[name] = member - return members - - -def read_tar_member(tar: tarfile.TarFile, member: tarfile.TarInfo) -> bytes: - if member.size > 1024 * 1024 * 1024: - raise ScanError("archive member is too large to inspect") - fileobj = tar.extractfile(member) - if fileobj is None: - raise ScanError("archive member could not be read") - return fileobj.read() - - -def open_archive(path: Path) -> tuple[tarfile.TarFile, dict[str, tarfile.TarInfo]]: - try: - tar = tarfile.open(path, "r:*") - except tarfile.TarError as error: - raise ScanError("image archive is not a readable tar archive") from error - return tar, safe_tar_members(tar) - - -def read_named_member( - tar: tarfile.TarFile, members: dict[str, tarfile.TarInfo], name: str -) -> bytes: - member = members.get(name) - if member is None: - raise ScanError("referenced archive member is missing") - if not member.isfile(): - raise ScanError("referenced archive member is not a file") - return read_tar_member(tar, member) - - -def parse_json(data: bytes, description: str) -> Any: - try: - return json.loads(data.decode("utf-8")) - except Exception as error: # noqa: BLE001 - raise ScanError(f"{description} is not valid JSON") from error - - -def maybe_decompress_layer(data: bytes, media_type: str | None) -> bytes: - if media_type and media_type.endswith("+zstd"): - raise ScanError("zstd-compressed layers are not supported") - if media_type and "gzip" in media_type: - try: - return gzip.decompress(data) - except Exception as error: # noqa: BLE001 - raise ScanError("gzip layer could not be decompressed") from error - if data.startswith(b"\x1f\x8b"): - try: - return gzip.decompress(data) - except Exception as error: # noqa: BLE001 - raise ScanError("gzip layer could not be decompressed") from error - return data - - -def scan_layer_bytes( - data: bytes, - needles: Iterable[bytes], - stats: Stats, - media_type: str | None = None, -) -> None: - layer = maybe_decompress_layer(data, media_type) - stats.layers += 1 - check_bytes(layer, needles, stats, "layer-tar") - - try: - with tarfile.open(fileobj=io.BytesIO(layer), mode="r:") as layer_tar: - for member in layer_tar: - stats.paths += 1 - check_path(member.name, needles, stats) - if member.linkname: - check_path(member.linkname, needles, stats) - if member.isfile(): - fileobj = layer_tar.extractfile(member) - if fileobj is None: - raise ScanError("layer file member could not be read") - check_bytes(fileobj.read(), needles, stats, "layer-file") - except tarfile.TarError as error: - raise ScanError("layer is not a readable tar archive") from error - - -def scan_metadata(data: bytes, needles: Iterable[bytes], stats: Stats) -> None: - stats.metadata += 1 - check_bytes(data, needles, stats, "metadata") - - -def scan_docker_archive(path: Path, needles: Iterable[bytes]) -> Stats: - stats = Stats() - tar, members = open_archive(path) - try: - if DOCKER_ARCHIVE_MANIFEST not in members: - raise ScanError("Docker archive manifest is missing") - manifest_bytes = read_named_member(tar, members, DOCKER_ARCHIVE_MANIFEST) - scan_metadata(manifest_bytes, needles, stats) - manifest = parse_json(manifest_bytes, "Docker archive manifest") - if not isinstance(manifest, list) or not manifest: - raise ScanError("Docker archive manifest has no images") - - for image in manifest: - if not isinstance(image, dict): - raise ScanError("Docker archive image entry is invalid") - config_name = image.get("Config") - layers = image.get("Layers") - if not isinstance(config_name, str) or not isinstance(layers, list) or not layers: - raise ScanError("Docker archive image entry is incomplete") - stats.images += 1 - config_bytes = read_named_member(tar, members, config_name) - scan_metadata(config_bytes, needles, stats) - parse_json(config_bytes, "Docker image config") - for layer_name in layers: - if not isinstance(layer_name, str): - raise ScanError("Docker archive layer reference is invalid") - layer_bytes = read_named_member(tar, members, layer_name) - scan_layer_bytes(layer_bytes, needles, stats) - finally: - tar.close() - - if stats.images == 0 or stats.layers == 0: - raise ScanError("Docker archive did not contain any complete images") - return stats - - -def blob_path_from_digest(digest: str) -> str: - if not digest.startswith("sha256:"): - raise ScanError("only sha256 OCI digests are supported") - hex_digest = digest.split(":", 1)[1] - if len(hex_digest) != 64 or any(ch not in "0123456789abcdef" for ch in hex_digest): - raise ScanError("OCI digest is malformed") - return f"blobs/sha256/{hex_digest}" - - -def verify_digest(data: bytes, digest: str) -> None: - expected = digest.split(":", 1)[1] - actual = hashlib.sha256(data).hexdigest() - if actual != expected: - raise ScanError("OCI blob digest mismatch") - - -def read_oci_blob( - tar: tarfile.TarFile, - members: dict[str, tarfile.TarInfo], - descriptor: dict[str, Any], -) -> bytes: - digest = descriptor.get("digest") - if not isinstance(digest, str): - raise ScanError("OCI descriptor is missing a digest") - data = read_named_member(tar, members, blob_path_from_digest(digest)) - verify_digest(data, digest) - expected_size = descriptor.get("size") - if expected_size is not None and expected_size != len(data): - raise ScanError("OCI descriptor size mismatch") - return data - - -def scan_oci_descriptor( - tar: tarfile.TarFile, - members: dict[str, tarfile.TarInfo], - descriptor: dict[str, Any], - needles: Iterable[bytes], - stats: Stats, - visited: set[str], -) -> None: - media_type = descriptor.get("mediaType") - if not isinstance(media_type, str): - raise ScanError("OCI descriptor mediaType is missing") - data = read_oci_blob(tar, members, descriptor) - stats.blobs += 1 - - digest = descriptor["digest"] - if digest in visited: - return - visited.add(digest) - - if media_type in INDEX_MEDIA_TYPES: - scan_metadata(data, needles, stats) - index = parse_json(data, "OCI image index") - descriptors = index.get("manifests") - if not isinstance(descriptors, list) or not descriptors: - raise ScanError("OCI index contains no manifests") - for child in descriptors: - if not isinstance(child, dict): - raise ScanError("OCI index child descriptor is invalid") - scan_oci_descriptor(tar, members, child, needles, stats, visited) - return - - if media_type in MANIFEST_MEDIA_TYPES: - scan_metadata(data, needles, stats) - manifest = parse_json(data, "OCI image manifest") - config_desc = manifest.get("config") - layer_descs = manifest.get("layers") - if not isinstance(config_desc, dict) or not isinstance(layer_descs, list): - raise ScanError("OCI manifest is incomplete") - - config_type = config_desc.get("mediaType") - config_data = read_oci_blob(tar, members, config_desc) - stats.blobs += 1 - if isinstance(config_type, str) and (config_type in CONFIG_MEDIA_TYPES or config_type.endswith("+json")): - stats.images += 1 if config_type in CONFIG_MEDIA_TYPES else 0 - scan_metadata(config_data, needles, stats) - else: - raise ScanError("OCI config mediaType is unsupported") - - for layer_desc in layer_descs: - if not isinstance(layer_desc, dict): - raise ScanError("OCI layer descriptor is invalid") - layer_type = layer_desc.get("mediaType") - if not isinstance(layer_type, str): - raise ScanError("OCI layer mediaType is missing") - layer_data = read_oci_blob(tar, members, layer_desc) - stats.blobs += 1 - if layer_type in TAR_LAYER_MEDIA_TYPES or ".tar" in layer_type: - scan_layer_bytes(layer_data, needles, stats, layer_type) - elif layer_type in JSON_BLOB_MEDIA_TYPES or layer_type.endswith("+json"): - scan_metadata(layer_data, needles, stats) - else: - raise ScanError("OCI layer mediaType is unsupported") - return - - if media_type in CONFIG_MEDIA_TYPES or media_type in JSON_BLOB_MEDIA_TYPES or media_type.endswith("+json"): - scan_metadata(data, needles, stats) - return - - raise ScanError("OCI descriptor mediaType is unsupported") - - -def scan_oci_archive(path: Path, needles: Iterable[bytes]) -> Stats: - stats = Stats() - tar, members = open_archive(path) - try: - if OCI_LAYOUT not in members or OCI_INDEX not in members: - raise ScanError("OCI layout markers are missing") - layout_bytes = read_named_member(tar, members, OCI_LAYOUT) - scan_metadata(layout_bytes, needles, stats) - index_bytes = read_named_member(tar, members, OCI_INDEX) - scan_metadata(index_bytes, needles, stats) - index = parse_json(index_bytes, "OCI root index") - descriptors = index.get("manifests") - if not isinstance(descriptors, list) or not descriptors: - raise ScanError("OCI root index contains no manifests") - visited: set[str] = set() - for descriptor in descriptors: - if not isinstance(descriptor, dict): - raise ScanError("OCI root descriptor is invalid") - scan_oci_descriptor(tar, members, descriptor, needles, stats, visited) - finally: - tar.close() - - if stats.images == 0 or stats.layers == 0: - raise ScanError("OCI archive did not contain any complete image layers") - return stats - - -def scan_auto_archive(path: Path, needles: Iterable[bytes]) -> Stats: - tar, members = open_archive(path) - try: - member_names = set(members) - finally: - tar.close() - if OCI_LAYOUT in member_names and OCI_INDEX in member_names: - return scan_oci_archive(path, needles) - if DOCKER_ARCHIVE_MANIFEST in member_names: - return scan_docker_archive(path, needles) - raise ScanError("archive is neither Docker save nor OCI layout format") - - -def scan_docker_image(image: str, needles: Iterable[bytes]) -> Stats: - if shutil.which("docker") is None: - raise ScanError("docker CLI is not available") - with tempfile.TemporaryDirectory(prefix="image-secret-scan.") as tmp: - archive = Path(tmp) / "image.tar" - try: - subprocess.run( - ["docker", "image", "save", "--output", str(archive), image], - check=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - ) - except subprocess.CalledProcessError as error: - raise ScanError("docker image could not be exported for scanning") from error - return scan_docker_archive(archive, needles) - - -def add_blob(members: dict[str, bytes], payload: bytes) -> dict[str, Any]: - digest = hashlib.sha256(payload).hexdigest() - members[f"blobs/sha256/{digest}"] = payload - return {"mediaType": "application/vnd.oci.image.manifest.v1+json", "digest": f"sha256:{digest}", "size": len(payload)} - - -def make_layer(files: dict[str, bytes]) -> bytes: - raw = io.BytesIO() - with tarfile.open(fileobj=raw, mode="w") as tar: - for name, payload in files.items(): - info = tarfile.TarInfo(name=name) - info.size = len(payload) - info.mode = 0o600 - tar.addfile(info, io.BytesIO(payload)) - return gzip.compress(raw.getvalue()) - - -def make_oci_fixture( - path: Path, - *, - first_files: dict[str, bytes] | None = None, - second_files: dict[str, bytes] | None = None, -) -> None: - blobs: dict[str, bytes] = {} - - config = json.dumps( - { - "architecture": "amd64", - "os": "linux", - "config": {"Env": ["PATH=/usr/local/bin"]}, - "rootfs": {"type": "layers", "diff_ids": []}, - "history": [{"created_by": "fixture"}], - }, - separators=(",", ":"), - ).encode() - config_digest = hashlib.sha256(config).hexdigest() - blobs[f"blobs/sha256/{config_digest}"] = config - config_desc = { - "mediaType": "application/vnd.oci.image.config.v1+json", - "digest": f"sha256:{config_digest}", - "size": len(config), - } - - def add_layer(files: dict[str, bytes]) -> dict[str, Any]: - layer = make_layer(files) - layer_digest = hashlib.sha256(layer).hexdigest() - blobs[f"blobs/sha256/{layer_digest}"] = layer - return { - "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": f"sha256:{layer_digest}", - "size": len(layer), - } - - layer_descs = [add_layer(first_files or {"app/ok.txt": b"clean"})] - if second_files is not None: - layer_descs.append(add_layer(second_files)) - - manifest = json.dumps( - { - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": config_desc, - "layers": layer_descs, - }, - separators=(",", ":"), - ).encode() - manifest_digest = hashlib.sha256(manifest).hexdigest() - blobs[f"blobs/sha256/{manifest_digest}"] = manifest - - nested_index = json.dumps( - { - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [ - { - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": f"sha256:{manifest_digest}", - "size": len(manifest), - "platform": {"os": "linux", "architecture": "amd64"}, - } - ], - }, - separators=(",", ":"), - ).encode() - nested_digest = hashlib.sha256(nested_index).hexdigest() - blobs[f"blobs/sha256/{nested_digest}"] = nested_index - - root_index = json.dumps( - { - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.index.v1+json", - "manifests": [ - { - "mediaType": "application/vnd.oci.image.index.v1+json", - "digest": f"sha256:{nested_digest}", - "size": len(nested_index), - } - ], - }, - separators=(",", ":"), - ).encode() - - with tarfile.open(path, mode="w") as tar: - for name, payload in {"oci-layout": b'{"imageLayoutVersion":"1.0.0"}', "index.json": root_index, **blobs}.items(): - info = tarfile.TarInfo(name=name) - info.size = len(payload) - info.mode = 0o600 - tar.addfile(info, io.BytesIO(payload)) - - -def run_generator_for_self_test(config: Path, output: Path) -> None: - generator = Path(__file__).with_name("create-sfw-image-scan-needles.py") - env = { - **os.environ, - "SFW_BUN_CONFIG_FILE": str(config), - "SFW_IMAGE_SCAN_NEEDLES": str(output), - } - result = subprocess.run( - [sys.executable, str(generator)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - if result.returncode != 0: - raise ScanError("self-test needle generator failed") - - -def expect_generator_failure(config: Path, output: Path, protected_values: Iterable[str]) -> None: - generator = Path(__file__).with_name("create-sfw-image-scan-needles.py") - env = { - **os.environ, - "SFW_BUN_CONFIG_FILE": str(config), - "SFW_IMAGE_SCAN_NEEDLES": str(output), - } - result = subprocess.run( - [sys.executable, str(generator)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - env=env, - ) - if result.returncode == 0: - raise ScanError("self-test unsupported generator config did not fail closed") - if output.exists(): - raise ScanError("self-test generator failure left a needle file behind") - diagnostics = result.stdout + result.stderr - if any(value and value in diagnostics for value in protected_values): - raise ScanError("self-test generator diagnostics disclosed a protected value") - - -def expect_cli_scan_failure( - archive: Path, - needle_file: Path, - protected_values: Iterable[str], -) -> None: - result = subprocess.run( - [sys.executable, __file__, "--oci-archive", str(archive), "--needles-file", str(needle_file)], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - if result.returncode != 1: - raise ScanError("self-test leaky fixture did not fail with forbidden material") - diagnostics = result.stdout + result.stderr - if any(value and value in diagnostics for value in protected_values): - raise ScanError("self-test scanner diagnostics disclosed a protected value") - - -def run_self_test() -> None: - token = "SYNTHETIC_SOCKET_TOKEN_SCANNER_20260909" - endpoint = "https://socket-firewall.workos.dev/" - host = "socket-firewall.workos.dev" - begin = "# >>> workos-sfw >>>" - end = "# <<< workos-sfw <<<" - with tempfile.TemporaryDirectory(prefix="image-secret-scan-self-test.") as tmp: - tmp_path = Path(tmp) - config = tmp_path / ".bunfig.toml" - config.write_text( - f"\n{begin}\n[install]\nregistry = {{ url = \"{endpoint}\", token = \"{token}\" }}\n{end}\n", - encoding="utf-8", - ) - needle_file = tmp_path / "needles.json" - run_generator_for_self_test(config, needle_file) - needles = load_needles(needle_file) - if b"registry" in needles or b"url" in needles or b"token" in needles: - raise ScanError("self-test generator produced a bare TOML key needle") - for required in (token.encode(), endpoint.encode(), host.encode()): - if required not in needles: - raise ScanError("self-test generator did not preserve required scan values") - - clean = tmp_path / "clean.oci.tar" - make_oci_fixture( - clean, - first_files={ - "app/registry-map.js": b"const registry = new Map();\n", - "app/node_modules/npm-registry-fetch/index.js": b"module.exports = 'registry';\n", - }, - second_files={"app/other.txt": b"clean"}, - ) - clean_stats = scan_oci_archive(clean, needles) - if clean_stats.hit_count: - raise ScanError("self-test clean registry-word fixture produced a match") - - credential = tmp_path / "credential.oci.tar" - make_oci_fixture(credential, first_files={"app/token.txt": token.encode()}) - credential_stats = scan_oci_archive(credential, needles) - if credential_stats.hit_count == 0: - raise ScanError("self-test did not catch a credential leak") - - endpoint_leak = tmp_path / "endpoint.oci.tar" - make_oci_fixture(endpoint_leak, first_files={"app/endpoint.txt": endpoint.encode()}) - endpoint_stats = scan_oci_archive(endpoint_leak, needles) - if endpoint_stats.hit_count == 0: - raise ScanError("self-test did not catch a Socket endpoint leak") - - deleted_lower_layer = tmp_path / "deleted-lower-layer.oci.tar" - make_oci_fixture( - deleted_lower_layer, - first_files={ - "app/.bunfig.toml": ( - f'registry = {{ url = "{endpoint}", token = "{token}" }}\n' - ).encode(), - }, - second_files={"app/.wh..bunfig.toml": b""}, - ) - deleted_stats = scan_oci_archive(deleted_lower_layer, needles) - if deleted_stats.hit_count == 0: - raise ScanError("self-test did not catch a deleted lower-layer leak") - expect_cli_scan_failure( - deleted_lower_layer, - needle_file, - [token, endpoint, host, "app/.bunfig.toml", str(needle_file), str(config)], - ) - - malformed = tmp_path / "malformed.oci.tar" - with tarfile.open(malformed, mode="w") as tar: - payload = b'{"schemaVersion":2,"manifests":[]}' - for name, data in {"oci-layout": b'{"imageLayoutVersion":"1.0.0"}', "index.json": payload}.items(): - info = tarfile.TarInfo(name=name) - info.size = len(data) - tar.addfile(info, io.BytesIO(data)) - try: - scan_oci_archive(malformed, needles) - except ScanError: - pass - else: - raise ScanError("self-test malformed OCI layout did not fail closed") - - bad_config = tmp_path / "bad.bunfig.toml" - bad_output = tmp_path / "bad-needles.json" - bad_config.write_text( - f"{begin}\n[install]\nregistry = {{ url = \"{endpoint}\", token = \"{token}\" }}\nextra = true\n{end}\n", - encoding="utf-8", - ) - expect_generator_failure(bad_config, bad_output, [token, endpoint, host, str(bad_config)]) - - print("image secret scanner self-test passed") - - - -def print_success(stats: Stats) -> None: - print( - "image secret scan passed: " - f"images={stats.images} layers={stats.layers} paths={stats.paths} " - f"metadata={stats.metadata}" - ) - - -def print_failure(stats: Stats) -> None: - categories = ",".join(f"{key}:{stats.hits[key]}" for key in sorted(stats.hits)) - print( - "error: forbidden Docker image material detected; " - f"hitCount={stats.hit_count}; categories={categories}; diagnostics=redacted", - file=sys.stderr, - ) - - -def main() -> int: - parser = argparse.ArgumentParser(description="Scan Docker/OCI images for private install config leaks") - parser.add_argument("--needles-file", type=Path, help="private JSON needle file created in runner temp") - parser.add_argument("--docker-image", help="local Docker image tag to export and scan") - parser.add_argument("--docker-archive", type=Path, help="docker save archive to scan") - parser.add_argument("--oci-archive", type=Path, help="OCI image-layout archive to scan") - parser.add_argument("--self-test", action="store_true", help="run scanner controls and exit") - args = parser.parse_args() - - try: - if args.self_test: - run_self_test() - return 0 - - targets = [args.docker_image, args.docker_archive, args.oci_archive] - if sum(value is not None for value in targets) != 1: - raise ScanError("choose exactly one image/archive target") - if args.needles_file is None: - raise ScanError("--needles-file is required") - needles = load_needles(args.needles_file) - - if args.docker_image: - stats = scan_docker_image(args.docker_image, needles) - elif args.docker_archive: - stats = scan_docker_archive(args.docker_archive, needles) - else: - stats = scan_oci_archive(args.oci_archive, needles) - - if stats.hit_count: - print_failure(stats) - return 1 - print_success(stats) - return 0 - except ScanError as error: - print(f"error: image secret scan failed closed: {error}", file=sys.stderr) - return 2 - - -if __name__ == "__main__": - raise SystemExit(main()) From bfaab7489e7b1ac06b4ebe9ca30ade75634a2107 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 10 Sep 2026 18:15:40 -0400 Subject: [PATCH 7/7] pi: address PR review feedback (#103) --- .github/workflows/release.yml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 90b89c1..2747e79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,8 +77,21 @@ jobs: - name: Install run: bun install --frozen-lockfile + - name: Check out release tooling + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.workflow_sha }} + path: .release-tools + persist-credentials: false + sparse-checkout: scripts/build-binaries.sh + sparse-checkout-cone-mode: false + - name: Build standalone binary - run: scripts/build-binaries.sh "${{ inputs.tag_name }}" "${{ matrix.target }}" "${{ matrix.asset }}" + env: + TAG_NAME: ${{ inputs.tag_name }} + TARGET: ${{ matrix.target }} + ASSET: ${{ matrix.asset }} + run: .release-tools/scripts/build-binaries.sh "$TAG_NAME" "$TARGET" "$ASSET" - name: Teardown Socket Firewall if: ${{ always() && steps.sfw.outputs.active == 'true' }}