From a20f6f35d91690925b461b15bc5dd78a85118fc3 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 17:54:14 +0000 Subject: [PATCH 1/6] feat: Publish `@braintrust/bt` on npm --- .github/workflows/release.yml | 31 +- .github/workflows/tests.yml | 20 + .gitignore | 1 + CONTRIBUTING.md | 29 ++ README.md | 17 + npm/bt/README.md | 68 +++ npm/bt/bin/bt | 41 ++ npm/bt/package.json | 40 ++ npm/bt/scripts/bt-helper.js | 161 +++++++ npm/bt/scripts/install.js | 181 +++++++ npm/platforms/README.md | 5 + npm/platforms/bt-darwin-arm64/package.json | 22 + npm/platforms/bt-darwin-x64/package.json | 22 + npm/platforms/bt-linux-arm64/package.json | 25 + npm/platforms/bt-linux-x64-musl/package.json | 25 + npm/platforms/bt-linux-x64/package.json | 25 + npm/platforms/bt-win32-arm64/package.json | 22 + npm/platforms/bt-win32-x64/package.json | 22 + npm/scripts/build-platform-packages.mjs | 87 ++-- npm/targets.json | 59 +-- npm/tests/packages.test.mjs | 306 ++++++++++++ npm/tests/runtime.test.cjs | 472 +++++++++++++++++++ 22 files changed, 1589 insertions(+), 92 deletions(-) create mode 100644 npm/bt/README.md create mode 100755 npm/bt/bin/bt create mode 100644 npm/bt/package.json create mode 100644 npm/bt/scripts/bt-helper.js create mode 100644 npm/bt/scripts/install.js create mode 100644 npm/platforms/README.md create mode 100644 npm/platforms/bt-darwin-arm64/package.json create mode 100644 npm/platforms/bt-darwin-x64/package.json create mode 100644 npm/platforms/bt-linux-arm64/package.json create mode 100644 npm/platforms/bt-linux-x64-musl/package.json create mode 100644 npm/platforms/bt-linux-x64/package.json create mode 100644 npm/platforms/bt-win32-arm64/package.json create mode 100644 npm/platforms/bt-win32-x64/package.json create mode 100644 npm/tests/packages.test.mjs create mode 100644 npm/tests/runtime.test.cjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99701d93..4847d73c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -576,7 +576,7 @@ jobs: --archives-dir archives \ --out-dir npm/dist - - name: Publish platform packages + - name: Publish platform packages and standalone CLI shell: bash run: | set -euo pipefail @@ -586,6 +586,8 @@ jobs: echo "No npm platform packages found in npm/dist" >&2 exit 1 fi + # Publish the wrapper last so every exact optional dependency exists. + package_dirs+=(npm/dist/bt) for dir in "${package_dirs[@]}"; do name="$(node -p "require('./$dir/package.json').name")" version="$(node -p "require('./$dir/package.json').version")" @@ -597,5 +599,30 @@ jobs: continue fi echo "Publishing $name@$version" - (cd "$dir" && npm publish --access public) + (cd "$dir" && npm publish --access public --tag latest) done + + smoke-npm: + needs: + - plan + - publish-npm + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + permissions: + contents: read + steps: + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Install and run the published CLI + shell: bash + env: + RELEASE_VERSION: ${{ needs.plan.outputs.release-version }} + run: | + set -euo pipefail + npm install --global --ignore-scripts "@braintrust/bt@${RELEASE_VERSION}" + test "$(bt --version)" = "bt ${RELEASE_VERSION}" + test "$(npx --yes "@braintrust/bt@${RELEASE_VERSION}" --version)" = "bt ${RELEASE_VERSION}" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e93c647a..35e47b25 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,26 @@ on: branches: [main] jobs: + npm-tests: + name: npm-tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + permissions: + contents: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "22" + - name: Test npm launcher and installer + run: node --test npm/tests/runtime.test.cjs + - name: Test npm package generation and installation + if: runner.os == 'Linux' + run: node --test npm/tests/packages.test.mjs + core-tests: name: core-tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 984c9820..4f6de071 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ Thumbs.db # Node node_modules/ +/npm/dist/ tests/evals/js/eval-bun/test-data.txt # Python diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b3c2333..a9542028 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,6 +140,35 @@ Notes: - The workflow publishes an immutable tag: `canary--`. - It also updates a moving branch tag: `canary-` when the run is for the latest commit on that branch. +## npm Releases + +Stable releases publish seven platform binary packages, then the standalone `@braintrust/bt` package, from the `publish-npm` job in `release.yml`. +The wrapper and its optional dependencies use the same exact CLI release version. +Installation follows the [optional dependencies plus postinstall fallback pattern](https://blog.sentry.io/publishing-binaries-on-npm/). +The builder also embeds SHA-256 checksums of the release binaries in the wrapper, following Sentry CLI's checksum approach; fallback downloads must match before they are installed. +Existing published versions are skipped so a partially completed publishing job can be retried. +Canary workflows do not publish npm packages. + +Package manifests are checked in under `npm/bt` and `npm/platforms`, including the wrapper's explicit optional dependencies and each platform's OS, CPU, and libc constraints. +Their `0.0.0` versions are placeholders; the release script stamps the release version into all manifests and dependency pins, copies the binaries, and calculates checksums. +`npm/targets.json` only maps Rust targets to platform package directories. +Publish the prepared packages under `npm/dist`, not the source directories. +To prepare packages from downloaded release archives: + +```bash +node npm/scripts/build-platform-packages.mjs --version --archives-dir +``` + +Before the first standalone release, an npm organization maintainer must publish the generated `@braintrust/bt` package with their authenticated npm account after its matching platform packages are available. +For this one-time local bootstrap, run `npm publish ./npm/dist/bt --access public --provenance=false`; subsequent CI publishes generate provenance through OIDC. +Then configure its npm trusted publisher with organization `braintrustdata`, repository `bt`, workflow filename `release.yml`, and environment `npm-publish`, allowing direct publishing. +Subsequent releases use the existing workflow's OIDC authentication without an npm publishing token. +Make the standalone package available before releasing the SDK change that removes its bundled CLI. + +Run packaging tests with `node --test npm/tests/*.test.cjs npm/tests/*.test.mjs`. +The archive builder tests require `tar`, `unzip`, and Python 3 and run on Linux in CI. +Runtime tests also run on macOS and Windows. + ## Windows Release Signing Release workflows Authenticode-sign Windows artifacts via [Azure Trusted Signing](https://learn.microsoft.com/azure/trusted-signing/) (the `sign-windows-artifacts` composite action). Signing runs when these GitHub Actions repository secrets are configured: diff --git a/README.md b/README.md index 450c386a..3e24c343 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,19 @@ ## Install +### npm + +```bash +npm install -g @braintrust/bt +bt --version +``` + +For a project-local installation, use `npm install --save-dev @braintrust/bt` and run `npx bt`. +To run without adding a project dependency, use `npx @braintrust/bt --help`. +Append `@` to the package name to install a specific stable CLI release. +The CLI is installed separately from the `braintrust` JavaScript SDK. +See the [npm package README](npm/bt/README.md) for supported platforms and fallback download behavior. + ### Unix (macOS / Linux) ```bash @@ -94,6 +107,8 @@ shasum -a 256 -c "bt-.tar.gz.sha256" ## Update +For npm installations, use `npm install -g @braintrust/bt@latest` globally or `npm install --save-dev @braintrust/bt@latest` in a project. + `bt` can update itself when installed via the official installer. ```bash @@ -109,6 +124,8 @@ bt update --channel canary ## Uninstall +For npm installations, use `npm uninstall -g @braintrust/bt` globally or `npm uninstall @braintrust/bt` in a project. + Unix-like systems: ```bash diff --git a/npm/bt/README.md b/npm/bt/README.md new file mode 100644 index 00000000..b619a975 --- /dev/null +++ b/npm/bt/README.md @@ -0,0 +1,68 @@ +# Braintrust CLI (`bt`) + +Install the CLI globally: + +```bash +npm install -g @braintrust/bt +bt --version +``` + +Or add it to a project: + +```bash +npm install --save-dev @braintrust/bt +npx bt --help +``` + +Run without adding a project dependency: + +```bash +npx @braintrust/bt --help +``` + +Use `@braintrust/bt@` to select a specific stable CLI release. +The CLI is distributed separately from the `braintrust` JavaScript SDK. +See the [CLI documentation](https://github.com/braintrustdata/bt#readme) for commands and configuration. + +## Installation behavior + +The package installs the matching `@braintrust/bt-*` binary as an optional dependency. +Prebuilt binaries are available for macOS arm64/x64, Linux glibc arm64/x64, Linux musl x64, and Windows arm64/x64. +`bt eval` currently supports Linux and macOS only. + +If optional dependencies are omitted, the postinstall script attempts to download the matching binary from the public npm registry. +The fallback binary is verified against a SHA-256 checksum shipped with this package before it is installed. +Download, checksum, and file-write failures fail installation with a nonzero exit code. +Unsupported platforms and missing release metadata also fail installation. +The launcher reports an error if no binary is available when invoked. +When install scripts are disabled, optional dependencies must be installed for the CLI to work. + +| Optional dependencies | Install scripts | Result | +| --------------------- | --------------- | ------------------------------------------------------------------- | +| Enabled | Enabled | Use the platform package; no fallback download. | +| Enabled | Disabled | Use the platform package. | +| Disabled | Enabled | Download and verify the fallback binary. | +| Disabled | Disabled | Installation succeeds, but running `bt` reports the missing binary. | + +If both features were disabled, reinstall with `npm install --include=optional @braintrust/bt` (add `--global` for a global installation). +When moving a project between operating systems or architectures, reinstall dependencies on the destination instead of copying `node_modules`. + +The launcher and installer retain two compatibility overrides from the JavaScript SDK: + +- `BT_BINARY_PATH` selects an existing executable instead of the packaged binary and skips the postinstall download. +- `BT_SKIP_DOWNLOAD=1` disables the postinstall fallback download. + +These apply before the native CLI runs; native CLI configuration remains available through its flags and environment variables. + +## Update and uninstall + +Update a global installation: + +```bash +npm install -g @braintrust/bt@latest +``` + +For a project installation, use `npm install --save-dev @braintrust/bt@latest`. +Use npm to update npm installations; `bt update` is for official installer installations. + +Uninstall globally with `npm uninstall -g @braintrust/bt`, or from a project with `npm uninstall @braintrust/bt`. diff --git a/npm/bt/bin/bt b/npm/bt/bin/bt new file mode 100755 index 00000000..5741f84d --- /dev/null +++ b/npm/bt/bin/bt @@ -0,0 +1,41 @@ +#!/usr/bin/env node +"use strict"; + +// Runtime launcher for the `bt` CLI. Resolves the platform-specific binary +// (installed via the matching `@braintrust/bt-*` optionalDependency, or +// downloaded by `scripts/install.js` as a fallback) and forwards argv, +// stdio, signals, and the exit code. + +const childProcess = require("node:child_process"); +const { getBinaryPath } = require("../scripts/bt-helper"); + +let binaryPath; +try { + binaryPath = getBinaryPath(); +} catch (err) { + console.error(err.message); + process.exit(1); +} + +const child = childProcess + .spawn(binaryPath, process.argv.slice(2), { + stdio: "inherit", + windowsHide: true, + }) + .on("error", (err) => { + console.error(err); + process.exit(1); + }) + .on("exit", (code, signal) => { + if (signal) { + // Detach our forwarding listener so the re-raised signal hits Node's + // default handler; otherwise we'd intercept it again and exit 0. + process.removeAllListeners(signal); + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); + }); + +process.on("SIGTERM", () => child.kill("SIGTERM")); +process.on("SIGINT", () => child.kill("SIGINT")); diff --git a/npm/bt/package.json b/npm/bt/package.json new file mode 100644 index 00000000..0e758060 --- /dev/null +++ b/npm/bt/package.json @@ -0,0 +1,40 @@ +{ + "name": "@braintrust/bt", + "version": "0.0.0", + "private": true, + "description": "The Braintrust command line interface", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "bin": { + "bt": "./bin/bt" + }, + "files": [ + "checksums.json", + "bin/bt", + "scripts/bt-helper.js", + "scripts/install.js" + ], + "scripts": { + "postinstall": "node ./scripts/install.js" + }, + "preferUnplugged": true, + "publishConfig": { + "access": "public", + "registry": "https://registry.npmjs.org/", + "provenance": true + }, + "optionalDependencies": { + "@braintrust/bt-darwin-arm64": "0.0.0", + "@braintrust/bt-darwin-x64": "0.0.0", + "@braintrust/bt-linux-arm64": "0.0.0", + "@braintrust/bt-linux-x64": "0.0.0", + "@braintrust/bt-linux-x64-musl": "0.0.0", + "@braintrust/bt-win32-arm64": "0.0.0", + "@braintrust/bt-win32-x64": "0.0.0" + } +} diff --git a/npm/bt/scripts/bt-helper.js b/npm/bt/scripts/bt-helper.js new file mode 100644 index 00000000..3f057f88 --- /dev/null +++ b/npm/bt/scripts/bt-helper.js @@ -0,0 +1,161 @@ +"use strict"; + +// Shared helpers for locating the `bt` binary. Used by both `bin/bt` +// (the runtime launcher) and `scripts/install.js` (the postinstall +// fallback downloader). + +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); + +const BINARY_DISTRIBUTIONS = [ + { packageName: "@braintrust/bt-darwin-arm64", subpath: "bin/bt" }, + { packageName: "@braintrust/bt-darwin-x64", subpath: "bin/bt" }, + { packageName: "@braintrust/bt-linux-arm64", subpath: "bin/bt" }, + { packageName: "@braintrust/bt-linux-x64", subpath: "bin/bt" }, + { packageName: "@braintrust/bt-linux-x64-musl", subpath: "bin/bt" }, + { packageName: "@braintrust/bt-win32-arm64", subpath: "bin/bt.exe" }, + { packageName: "@braintrust/bt-win32-x64", subpath: "bin/bt.exe" }, +]; + +function detectLibc() { + if (process.platform !== "linux") return null; + try { + const report = process.report && process.report.getReport(); + if (report && report.header && report.header.glibcVersionRuntime) { + return "glibc"; + } + return "musl"; + } catch { + return "glibc"; + } +} + +function binaryName() { + return process.platform === "win32" ? "bt.exe" : "bt"; +} + +function getDistributionForThisPlatform() { + const arch = os.arch(); + const platform = os.platform(); + const subpath = `bin/${binaryName()}`; + + let packageName; + if (platform === "darwin") { + if (arch === "arm64") packageName = "@braintrust/bt-darwin-arm64"; + else if (arch === "x64") packageName = "@braintrust/bt-darwin-x64"; + } else if (platform === "linux") { + const libc = detectLibc(); + if (arch === "arm64" && libc === "glibc") { + packageName = "@braintrust/bt-linux-arm64"; + } else if (arch === "x64") { + packageName = + libc === "musl" + ? "@braintrust/bt-linux-x64-musl" + : "@braintrust/bt-linux-x64"; + } + } else if (platform === "win32") { + if (arch === "arm64") packageName = "@braintrust/bt-win32-arm64"; + else if (arch === "x64") packageName = "@braintrust/bt-win32-x64"; + } + + return { packageName, subpath }; +} + +function throwUnsupportedPlatformError() { + throw new Error( + `Unsupported operating system or architecture! The bt CLI does not work on ${process.platform}-${process.arch}. + +bt supports: +- macOS (darwin) on arm64 and x64 +- Linux glibc on arm64 and x64; Linux musl on x64 +- Windows on arm64 and x64`, + ); +} + +// Constructed indirectly so bundlers (e.g. @vercel/nft) don't statically +// detect the fallback binary path as an asset to trace. +function getFallbackBinaryPath() { + const parts = [__dirname, binaryName()]; + return path.resolve(...parts); +} + +function resolvePackageBinary(packageName, subpath) { + const binaryPath = require.resolve(`${packageName}/${subpath}`); + // Read the manifest beside this binary, which may belong to a hoisted + // dependency rather than the version pinned by this wrapper. + const installed = require( + path.resolve(path.dirname(binaryPath), "..", "package.json"), + ); + const expectedVersion = + require("../package.json").optionalDependencies[packageName]; + if (installed.version !== expectedVersion) { + throw new Error( + `Expected ${packageName}@${expectedVersion}, but found ${installed.version}. Reinstall @braintrust/bt with optional dependencies enabled.`, + ); + } + return binaryPath; +} + +function getBinaryPath() { + // npm launcher compatibility: this override selects the executable before + // clap can run, so it cannot be configured by the Rust CLI. + if (process.env.BT_BINARY_PATH) { + return process.env.BT_BINARY_PATH; + } + + const { packageName, subpath } = getDistributionForThisPlatform(); + + if (packageName === undefined) { + throwUnsupportedPlatformError(); + } + + // Prefer the optional dep so a stale fallback from a prior + // `--omit=optional` install can't shadow a newer optional dep on upgrade. + try { + return resolvePackageBinary(packageName, subpath); + } catch (err) { + const fallbackBinaryPath = getFallbackBinaryPath(); + if (fs.existsSync(fallbackBinaryPath)) { + return fallbackBinaryPath; + } + if (err.code !== "MODULE_NOT_FOUND") throw err; + + const otherInstalled = BINARY_DISTRIBUTIONS.find((dist) => { + try { + require.resolve(`${dist.packageName}/${dist.subpath}`); + return true; + } catch { + return false; + } + }); + + // Error messages inspired by esbuild: + // https://github.com/evanw/esbuild/blob/f3d535262e3998d845d0f102b944ecd5a9efda57/lib/npm/node-platform.ts#L150 + if (otherInstalled) { + throw new Error( + `bt binary for this platform/architecture not found! + +The "${otherInstalled.packageName}" package is installed, but for the current platform you should have the "${packageName}" package installed instead. This usually happens if "@braintrust/bt" is installed on one platform (for example macOS or Windows) and the "node_modules" folder is then reused on another (for example Linux in Docker). + +To fix this, avoid copying the "node_modules" folder, and instead freshly install your dependencies on the target system. You can also configure your package manager to install the right package. For example, yarn has the "supportedArchitectures" feature: https://yarnpkg.com/configuration/yarnrc/#supportedArchitecture.`, + ); + } + + throw new Error( + `bt binary for this platform/architecture not found! + +It seems like none of the "@braintrust/bt" package's optional dependencies got installed. Please make sure your package manager is configured to install optional dependencies. If you are using npm, don't set the "--no-optional", "--ignore-optional", or "--omit=optional" flags. The "@braintrust/bt" package needs the "optionalDependencies" feature in order to install the bt binary.`, + ); + } +} + +module.exports = { + BINARY_DISTRIBUTIONS, + binaryName, + getBinaryPath, + getDistributionForThisPlatform, + getFallbackBinaryPath, + resolvePackageBinary, + throwUnsupportedPlatformError, +}; diff --git a/npm/bt/scripts/install.js b/npm/bt/scripts/install.js new file mode 100644 index 00000000..cd9acda3 --- /dev/null +++ b/npm/bt/scripts/install.js @@ -0,0 +1,181 @@ +"use strict"; + +// Postinstall script for the `bt` CLI binary. +// +// The native binary ships in a per-platform `@braintrust/bt-*` package listed +// in `optionalDependencies`; npm/pnpm install only the one matching the host. +// If a package manager is run with `--no-optional`, `--ignore-optional`, or +// `--omit=optional`, none of those packages get installed and `bt` would be +// unusable. As a workaround, we manually fetch the matching tarball from the +// npm registry and extract the binary so the launcher can find it. + +const fs = require("node:fs"); +const { createHash } = require("node:crypto"); +const https = require("node:https"); +const path = require("node:path"); +const zlib = require("node:zlib"); + +const helper = require("./bt-helper"); +const pkg = require("../package.json"); + +// Installation runs before the Rust CLI exists, so clap cannot own these +// download opt-outs. The launcher also honors BT_BINARY_PATH. +if (process.env.BT_SKIP_DOWNLOAD === "1") { + console.log( + "bt: skipping post-install binary download because BT_SKIP_DOWNLOAD=1 is set.", + ); + process.exit(0); +} + +if (process.env.BT_BINARY_PATH) { + console.log( + "bt: skipping post-install binary download because BT_BINARY_PATH is set.", + ); + process.exit(0); +} + +const { packageName, subpath } = helper.getDistributionForThisPlatform(); + +if (packageName === undefined) { + console.error( + `bt: no prebuilt binary available for ${process.platform}-${process.arch}; the bt CLI will not be available.`, + ); + process.exit(1); +} + +try { + helper.resolvePackageBinary(packageName, subpath); + // Optional dependency was installed successfully. Nothing to do. + process.exit(0); +} catch (err) { + // Fall through to the manual download path below. + console.log( + `bt: failed to locate the matching "${packageName}" package after installation: ${err.message} + +This can happen if you use an option to disable optional dependencies during installation, like "--no-optional", "--ignore-optional", or "--omit=optional". The "@braintrust/bt" package uses the "optionalDependencies" package.json feature to install the correct bt binary for your platform and operating system. This post-install script will now try to work around that by manually downloading the bt binary from the npm registry. If this fails, you need to remove the "--no-optional", "--ignore-optional", and "--omit=optional" flags for bt to work.`, + ); +} + +const version = (pkg.optionalDependencies || {})[packageName]; +if (!version) { + console.error( + `bt: cannot determine which version of "${packageName}" to download — it is not listed in the "@braintrust/bt" package's optionalDependencies. The bt CLI will not be available.`, + ); + process.exit(1); +} + +function fetchBuffer(url, redirectsRemaining = 5) { + return new Promise((resolve, reject) => { + https + .get(url, (response) => { + const { statusCode = 0, headers } = response; + if (statusCode >= 200 && statusCode < 300) { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => resolve(Buffer.concat(chunks))); + response.on("error", reject); + return; + } + if ( + statusCode >= 300 && + statusCode < 400 && + headers.location && + redirectsRemaining > 0 + ) { + response.resume(); + fetchBuffer(headers.location, redirectsRemaining - 1).then( + resolve, + reject, + ); + return; + } + response.resume(); + reject( + new Error( + `npm registry responded with status code ${statusCode} when downloading ${url}`, + ), + ); + }) + .on("error", reject); + }); +} + +// Extracts a single file from an uncompressed tar archive. Tar archives are +// organized in 512-byte blocks: a header block (file name in bytes 0-99, +// file size in bytes 124-135 as an octal string) followed by data blocks +// padded out to the next multiple of 512. +function extractFileFromTarball(tarball, target) { + let offset = 0; + while (offset + 512 <= tarball.length) { + const header = tarball.subarray(offset, offset + 512); + offset += 512; + const fileName = header.toString("utf-8", 0, 100).replace(/\0.*/g, ""); + if (!fileName) break; + const fileSize = parseInt( + header.toString("utf-8", 124, 136).replace(/\0.*/g, ""), + 8, + ); + if ( + !Number.isSafeInteger(fileSize) || + fileSize < 0 || + offset + fileSize > tarball.length + ) { + throw new Error(`Invalid or truncated tar entry: ${fileName}`); + } + if (fileName === target) { + return tarball.subarray(offset, offset + fileSize); + } + offset = Math.ceil((offset + fileSize) / 512) * 512; + } + return null; +} + +async function downloadFallback() { + // Like @sentry/cli, ship checksums with the wrapper so the fallback download + // can be checked against the release binaries without trusting the download. + const expectedChecksum = require("../checksums.json")[packageName]; + if (!expectedChecksum) { + throw new Error(`No release checksum found for ${packageName}@${version}`); + } + + // npm tarball URLs look like: + // https://registry.npmjs.org///-/-.tgz + // where is the unscoped package name. + const tarballName = packageName.split("/").pop(); + const url = `https://registry.npmjs.org/${packageName}/-/${tarballName}-${version}.tgz`; + console.log(`bt: downloading ${packageName}@${version} from ${url}`); + + const gzipped = await fetchBuffer(url); + const tarball = zlib.gunzipSync(gzipped); + const binary = extractFileFromTarball(tarball, `package/${subpath}`); + + if (!binary) { + throw new Error( + `could not find "package/${subpath}" inside ${packageName}@${version} tarball`, + ); + } + + const actualChecksum = createHash("sha256").update(binary).digest("hex"); + if (actualChecksum !== expectedChecksum) { + throw new Error(`Checksum validation failed for ${packageName}@${version}`); + } + + const fallbackBinaryPath = helper.getFallbackBinaryPath(); + fs.mkdirSync(path.dirname(fallbackBinaryPath), { recursive: true }); + const temporaryPath = `${fallbackBinaryPath}.${process.pid}.tmp`; + try { + fs.writeFileSync(temporaryPath, binary, { mode: 0o755 }); + fs.renameSync(temporaryPath, fallbackBinaryPath); + } finally { + fs.rmSync(temporaryPath, { force: true }); + } + console.log(`bt: installed fallback binary at ${fallbackBinaryPath}`); +} + +downloadFallback().catch((err) => { + console.error( + `bt: failed to download fallback binary for ${packageName}@${version}: ${err.message} +The bt CLI will not be available.`, + ); + process.exit(1); +}); diff --git a/npm/platforms/README.md b/npm/platforms/README.md new file mode 100644 index 00000000..654379f9 --- /dev/null +++ b/npm/platforms/README.md @@ -0,0 +1,5 @@ +# Braintrust CLI platform binary + +This package contains a prebuilt `bt` binary for the operating system and architecture specified in its `package.json`. +It is installed automatically as an optional dependency of [`@braintrust/bt`](https://www.npmjs.com/package/@braintrust/bt), which exposes the `bt` command. +Install `@braintrust/bt` instead of installing this package directly. diff --git a/npm/platforms/bt-darwin-arm64/package.json b/npm/platforms/bt-darwin-arm64/package.json new file mode 100644 index 00000000..359830a0 --- /dev/null +++ b/npm/platforms/bt-darwin-arm64/package.json @@ -0,0 +1,22 @@ +{ + "name": "@braintrust/bt-darwin-arm64", + "version": "0.0.0", + "description": "Prebuilt bt binary for darwin-arm64", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "darwin" + ], + "cpu": [ + "arm64" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-darwin-x64/package.json b/npm/platforms/bt-darwin-x64/package.json new file mode 100644 index 00000000..8109028c --- /dev/null +++ b/npm/platforms/bt-darwin-x64/package.json @@ -0,0 +1,22 @@ +{ + "name": "@braintrust/bt-darwin-x64", + "version": "0.0.0", + "description": "Prebuilt bt binary for darwin-x64", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "darwin" + ], + "cpu": [ + "x64" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-linux-arm64/package.json b/npm/platforms/bt-linux-arm64/package.json new file mode 100644 index 00000000..4d43ac7a --- /dev/null +++ b/npm/platforms/bt-linux-arm64/package.json @@ -0,0 +1,25 @@ +{ + "name": "@braintrust/bt-linux-arm64", + "version": "0.0.0", + "description": "Prebuilt bt binary for linux-arm64-glibc", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "linux" + ], + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-linux-x64-musl/package.json b/npm/platforms/bt-linux-x64-musl/package.json new file mode 100644 index 00000000..d6b37440 --- /dev/null +++ b/npm/platforms/bt-linux-x64-musl/package.json @@ -0,0 +1,25 @@ +{ + "name": "@braintrust/bt-linux-x64-musl", + "version": "0.0.0", + "description": "Prebuilt bt binary for linux-x64-musl", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-linux-x64/package.json b/npm/platforms/bt-linux-x64/package.json new file mode 100644 index 00000000..15f1de02 --- /dev/null +++ b/npm/platforms/bt-linux-x64/package.json @@ -0,0 +1,25 @@ +{ + "name": "@braintrust/bt-linux-x64", + "version": "0.0.0", + "description": "Prebuilt bt binary for linux-x64-glibc", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "linux" + ], + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-win32-arm64/package.json b/npm/platforms/bt-win32-arm64/package.json new file mode 100644 index 00000000..be9c8975 --- /dev/null +++ b/npm/platforms/bt-win32-arm64/package.json @@ -0,0 +1,22 @@ +{ + "name": "@braintrust/bt-win32-arm64", + "version": "0.0.0", + "description": "Prebuilt bt binary for win32-arm64", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "win32" + ], + "cpu": [ + "arm64" + ], + "preferUnplugged": true +} diff --git a/npm/platforms/bt-win32-x64/package.json b/npm/platforms/bt-win32-x64/package.json new file mode 100644 index 00000000..87a7e0a6 --- /dev/null +++ b/npm/platforms/bt-win32-x64/package.json @@ -0,0 +1,22 @@ +{ + "name": "@braintrust/bt-win32-x64", + "version": "0.0.0", + "description": "Prebuilt bt binary for win32-x64", + "homepage": "https://github.com/braintrustdata/bt", + "repository": { + "type": "git", + "url": "git+https://github.com/braintrustdata/bt.git" + }, + "license": "Apache-2.0", + "author": "Braintrust engineering ", + "files": [ + "bin/" + ], + "os": [ + "win32" + ], + "cpu": [ + "x64" + ], + "preferUnplugged": true +} diff --git a/npm/scripts/build-platform-packages.mjs b/npm/scripts/build-platform-packages.mjs index ff523ac4..048be6e1 100644 --- a/npm/scripts/build-platform-packages.mjs +++ b/npm/scripts/build-platform-packages.mjs @@ -1,17 +1,16 @@ #!/usr/bin/env node -// Builds the per-platform npm packages from cargo-dist release archives. +// Prepares checked-in npm packages using cargo-dist release archives. // // --version version to stamp into every package.json (required) // --archives-dir directory containing cargo-dist archives // (bt-.tar.gz / bt-.zip), required // --out-dir directory to write packages into (default: npm/dist) // -// Emits /bt-/ (one per target), each ready to `npm publish`. -// The `bt` command is exposed via the `braintrust` SDK, which lists these -// packages as optionalDependencies and ships a launcher that resolves the -// matching binary. +// Emits /bt-/ (one per target) and /bt/, each ready +// to `npm publish`. The @braintrust/bt package exposes the `bt` command. import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { chmodSync, cpSync, @@ -47,16 +46,23 @@ if (!existsSync(archivesDir)) throw new Error(`archives-dir not found: ${archivesDir}`); const targets = JSON.parse(readFileSync(join(NPM_DIR, "targets.json"), "utf8")); +const checksums = {}; if (existsSync(outDir)) rmSync(outDir, { recursive: true, force: true }); mkdirSync(outDir, { recursive: true }); // --- Per-platform packages --- -for (const [target, spec] of Object.entries(targets)) { - const archiveName = `bt-${target}.${spec.archiveExt}`; +for (const [target, platform] of Object.entries(targets)) { + const packageDir = join(NPM_DIR, "platforms", `bt-${platform}`); + const platformPkg = JSON.parse( + readFileSync(join(packageDir, "package.json"), "utf8"), + ); + const isWindows = platformPkg.os.includes("win32"); + const binaryName = isWindows ? "bt.exe" : "bt"; + const archiveName = `bt-${target}.${isWindows ? "zip" : "tar.gz"}`; const archive = join(archivesDir, archiveName); if (!existsSync(archive)) { - // Fail hard, don't skip: the SDK pins each package at an exact version, so a + // Fail hard, don't skip: the wrapper pins each package at an exact version, so a // missing platform would break installs for that platform at runtime. throw new Error(`Archive not found for ${target}: ${archive}`); } @@ -82,49 +88,54 @@ for (const [target, spec] of Object.entries(targets)) { throw new Error(`Unsupported archive: ${archive}`); } - const binPath = join(stagingDir, spec.bin); + const binPath = join(stagingDir, binaryName); if (!existsSync(binPath)) { - throw new Error(`Binary ${spec.bin} not found at ${binPath}`); + throw new Error(`Binary ${binaryName} not found at ${binPath}`); } - const pkgName = `@braintrust/bt-${spec.pkg}`; - const pkgOut = join(outDir, `bt-${spec.pkg}`); + checksums[platformPkg.name] = createHash("sha256") + .update(readFileSync(binPath)) + .digest("hex"); + const pkgOut = join(outDir, `bt-${platform}`); const pkgBin = join(pkgOut, "bin"); mkdirSync(pkgBin, { recursive: true }); - cpSync(binPath, join(pkgBin, spec.bin)); - if (spec.os !== "win32") chmodSync(join(pkgBin, spec.bin), 0o755); - - const platformPkg = { - name: pkgName, - version, - description: `Prebuilt bt binary for ${spec.os}-${spec.cpu}${spec.libc ? `-${spec.libc}` : ""}`, - homepage: "https://github.com/braintrustdata/bt", - repository: { - type: "git", - url: "git+https://github.com/braintrustdata/bt.git", - }, - license: "Apache-2.0", - author: "Braintrust engineering ", - files: ["bin/"], - os: [spec.os], - cpu: [spec.cpu], - preferUnplugged: true, - }; - if (spec.libc) platformPkg.libc = [spec.libc]; + cpSync(binPath, join(pkgBin, binaryName)); + if (!isWindows) chmodSync(join(pkgBin, binaryName), 0o755); + platformPkg.version = version; writeFileSync( join(pkgOut, "package.json"), JSON.stringify(platformPkg, null, 2) + "\n", ); - writeFileSync( - join(pkgOut, "README.md"), - `# ${pkgName}\n\nPrebuilt \`bt\` binary for ${spec.os}-${spec.cpu}${spec.libc ? ` (${spec.libc})` : ""}.\n\nInstalled automatically as an optional dependency of [\`braintrust\`](https://www.npmjs.com/package/braintrust), which exposes the \`bt\` command. Install that package instead.\n`, - ); + cpSync(join(NPM_DIR, "platforms", "README.md"), join(pkgOut, "README.md")); - console.log(`Built ${pkgName} -> ${pkgOut}`); + console.log(`Prepared ${platformPkg.name} -> ${pkgOut}`); } rmSync(join(outDir, ".staging"), { recursive: true, force: true }); -const expected = Object.keys(targets).length; +// --- Standalone CLI package --- +const wrapperOut = join(outDir, "bt"); +cpSync(join(NPM_DIR, "bt"), wrapperOut, { recursive: true }); +cpSync(join(NPM_DIR, "..", "LICENSE"), join(wrapperOut, "LICENSE")); +writeFileSync( + join(wrapperOut, "checksums.json"), + JSON.stringify(checksums, null, 2) + "\n", +); +chmodSync(join(wrapperOut, "bin", "bt"), 0o755); +const wrapperPkg = JSON.parse( + readFileSync(join(wrapperOut, "package.json"), "utf8"), +); +wrapperPkg.version = version; +delete wrapperPkg.private; +for (const dependency of Object.keys(wrapperPkg.optionalDependencies)) { + wrapperPkg.optionalDependencies[dependency] = version; +} +writeFileSync( + join(wrapperOut, "package.json"), + JSON.stringify(wrapperPkg, null, 2) + "\n", +); +console.log(`Built @braintrust/bt -> ${wrapperOut}`); + +const expected = Object.keys(targets).length + 1; console.log(`\nAll ${expected} packages written to ${outDir}`); diff --git a/npm/targets.json b/npm/targets.json index 21b6cda3..f33a3f91 100644 --- a/npm/targets.json +++ b/npm/targets.json @@ -1,54 +1,9 @@ { - "aarch64-apple-darwin": { - "pkg": "darwin-arm64", - "os": "darwin", - "cpu": "arm64", - "bin": "bt", - "archiveExt": "tar.gz" - }, - "x86_64-apple-darwin": { - "pkg": "darwin-x64", - "os": "darwin", - "cpu": "x64", - "bin": "bt", - "archiveExt": "tar.gz" - }, - "aarch64-unknown-linux-gnu": { - "pkg": "linux-arm64", - "os": "linux", - "cpu": "arm64", - "libc": "glibc", - "bin": "bt", - "archiveExt": "tar.gz" - }, - "x86_64-unknown-linux-gnu": { - "pkg": "linux-x64", - "os": "linux", - "cpu": "x64", - "libc": "glibc", - "bin": "bt", - "archiveExt": "tar.gz" - }, - "x86_64-unknown-linux-musl": { - "pkg": "linux-x64-musl", - "os": "linux", - "cpu": "x64", - "libc": "musl", - "bin": "bt", - "archiveExt": "tar.gz" - }, - "aarch64-pc-windows-msvc": { - "pkg": "win32-arm64", - "os": "win32", - "cpu": "arm64", - "bin": "bt.exe", - "archiveExt": "zip" - }, - "x86_64-pc-windows-msvc": { - "pkg": "win32-x64", - "os": "win32", - "cpu": "x64", - "bin": "bt.exe", - "archiveExt": "zip" - } + "aarch64-apple-darwin": "darwin-arm64", + "x86_64-apple-darwin": "darwin-x64", + "aarch64-unknown-linux-gnu": "linux-arm64", + "x86_64-unknown-linux-gnu": "linux-x64", + "x86_64-unknown-linux-musl": "linux-x64-musl", + "aarch64-pc-windows-msvc": "win32-arm64", + "x86_64-pc-windows-msvc": "win32-x64" } diff --git a/npm/tests/packages.test.mjs b/npm/tests/packages.test.mjs new file mode 100644 index 00000000..123a5148 --- /dev/null +++ b/npm/tests/packages.test.mjs @@ -0,0 +1,306 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { test } from "node:test"; + +const npmDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const targets = Object.fromEntries( + Object.entries( + JSON.parse(readFileSync(join(npmDir, "targets.json"), "utf8")), + ).map(([target, platform]) => { + const pkg = JSON.parse( + readFileSync( + join(npmDir, "platforms", `bt-${platform}`, "package.json"), + "utf8", + ), + ); + return [ + target, + { + pkg: platform, + os: pkg.os[0], + cpu: pkg.cpu[0], + libc: pkg.libc?.[0], + bin: pkg.os[0] === "win32" ? "bt.exe" : "bt", + archiveExt: pkg.os[0] === "win32" ? "zip" : "tar.gz", + }, + ]; + }), +); +const version = "1.2.3"; + +test( + "builds and installs the standalone package from release archives", + { skip: process.platform !== "linux" }, + async (t) => { + const root = mkdtempSync(join(tmpdir(), "bt-npm-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const archives = join(root, "archives"); + const out = join(root, "dist"); + mkdirSync(archives); + const stub = `#!/bin/sh\nprintf 'bt ${version}\\n'\n`; + for (const [target, spec] of Object.entries(targets)) { + const staging = join(root, `bt-${target}`); + mkdirSync(staging); + writeFileSync(join(staging, spec.bin), stub); + const archive = join(archives, `bt-${target}.${spec.archiveExt}`); + if (spec.archiveExt === "zip") { + execFileSync("python3", ["-m", "zipfile", "-c", archive, spec.bin], { + cwd: staging, + }); + } else { + execFileSync("tar", ["-czf", archive, "-C", root, `bt-${target}`]); + } + } + const args = [ + join(npmDir, "scripts/build-platform-packages.mjs"), + "--version", + version, + "--archives-dir", + archives, + "--out-dir", + out, + ]; + execFileSync(process.execPath, args); + const wrapper = JSON.parse( + readFileSync(join(out, "bt/package.json"), "utf8"), + ); + const env = { ...process.env, npm_config_cache: join(root, "cache") }; + delete env.BT_BINARY_PATH; + delete env.BT_SKIP_DOWNLOAD; + + await t.test( + "wrapper pins every platform and includes only the runtime files", + () => { + assert.equal(wrapper.name, "@braintrust/bt"); + assert.equal(wrapper.version, version); + assert.equal(wrapper.private, undefined); + assert.deepEqual(wrapper.bin, { bt: "./bin/bt" }); + assert.equal(wrapper.scripts.postinstall, "node ./scripts/install.js"); + assert.deepEqual( + wrapper.optionalDependencies, + Object.fromEntries( + Object.values(targets).map((spec) => [ + `@braintrust/bt-${spec.pkg}`, + version, + ]), + ), + ); + assert.equal(statSync(join(out, "bt/bin/bt")).mode & 0o777, 0o755); + assert.equal(existsSync(join(out, ".staging")), false); + const checksums = JSON.parse( + readFileSync(join(out, "bt/checksums.json"), "utf8"), + ); + assert.equal( + Object.keys(checksums).length, + Object.keys(targets).length, + ); + for (const spec of Object.values(targets)) { + const dir = join(out, `bt-${spec.pkg}`); + const pkg = JSON.parse( + readFileSync(join(dir, "package.json"), "utf8"), + ); + assert.equal(pkg.version, version); + assert.deepEqual(pkg.os, [spec.os]); + assert.deepEqual(pkg.cpu, [spec.cpu]); + assert.deepEqual(pkg.libc, spec.libc ? [spec.libc] : undefined); + assert.equal(readFileSync(join(dir, "bin", spec.bin), "utf8"), stub); + assert.equal( + checksums[pkg.name], + createHash("sha256").update(stub).digest("hex"), + ); + if (spec.os !== "win32") + assert.equal( + statSync(join(dir, "bin", spec.bin)).mode & 0o777, + 0o755, + ); + } + const [pack] = JSON.parse( + execFileSync( + "npm", + ["pack", "--dry-run", "--json", "--ignore-scripts"], + { cwd: join(out, "bt"), env, encoding: "utf8" }, + ), + ); + assert.deepEqual( + pack.files.map(({ path }) => path).sort(), + [ + "LICENSE", + "README.md", + "bin/bt", + "checksums.json", + "package.json", + "scripts/bt-helper.js", + "scripts/install.js", + ].sort(), + ); + }, + ); + + const helper = createRequire(import.meta.url)( + join(npmDir, "bt/scripts/bt-helper.js"), + ); + const hostName = helper + .getDistributionForThisPlatform() + .packageName.split("/")[1]; + const tarballs = []; + for (const name of ["bt", hostName]) { + const [packed] = JSON.parse( + execFileSync( + "npm", + ["pack", "--json", "--ignore-scripts", "--pack-destination", root], + { cwd: join(out, name), env, encoding: "utf8" }, + ), + ); + tarballs.push(join(root, packed.filename)); + } + + await t.test( + "local and global packed installs work with lifecycle scripts disabled", + () => { + const project = join(root, "project"); + mkdirSync(project); + writeFileSync( + join(project, "package.json"), + JSON.stringify({ name: "test-project", private: true }), + ); + const installArgs = [ + "install", + "--ignore-scripts", + "--offline", + "--no-audit", + "--no-fund", + ...tarballs, + ]; + execFileSync("npm", installArgs, { cwd: project, env, stdio: "pipe" }); + const output = execFileSync( + "npm", + ["exec", "--offline", "--", "bt", "--version"], + { cwd: project, env, encoding: "utf8" }, + ); + assert.equal(output.trim(), `bt ${version}`); + const prefix = join(root, "global"); + execFileSync("npm", [...installArgs, "--global", "--prefix", prefix], { + cwd: root, + env, + stdio: "pipe", + }); + assert.equal( + execFileSync(join(prefix, "bin/bt"), ["--version"], { + env, + encoding: "utf8", + }).trim(), + `bt ${version}`, + ); + }, + ); + + await t.test( + "disabling both optional dependencies and scripts gives an actionable launch error", + () => { + const project = join(root, "without-binary"); + mkdirSync(project); + writeFileSync( + join(project, "package.json"), + JSON.stringify({ name: "test-project", private: true }), + ); + execFileSync( + "npm", + [ + "install", + "--omit=optional", + "--ignore-scripts", + "--offline", + "--no-audit", + "--no-fund", + tarballs[0], + ], + { cwd: project, env, stdio: "pipe" }, + ); + const result = spawnSync( + join(project, "node_modules/.bin/bt"), + ["--version"], + { cwd: project, env, encoding: "utf8" }, + ); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match( + result.stderr, + /configured to install optional dependencies/, + ); + assert.equal( + existsSync(join(project, "node_modules/@braintrust/bt/scripts/bt")), + false, + ); + }, + ); + + await t.test( + "fallback installs and verifies the executable from a packed npm tarball", + () => { + const project = join(root, "fallback"); + cpSync(join(out, "bt"), project, { recursive: true }); + const preload = join(root, "registry.cjs"); + // Serve the real packed platform package without accessing the network. + writeFileSync( + preload, + ` + const assert = require("node:assert/strict"); + const { EventEmitter } = require("node:events"); + const { Readable } = require("node:stream"); + const tarball = require("node:fs").readFileSync(${JSON.stringify(tarballs[1])}); + require("node:https").get = (url, callback) => { + assert.equal(url, ${JSON.stringify(`https://registry.npmjs.org/@braintrust/${hostName}/-/${hostName}-${version}.tgz`)}); + const response = Readable.from([tarball]); + response.statusCode = 200; + response.headers = {}; + process.nextTick(() => callback(response)); + return new EventEmitter(); + }; + `, + ); + execFileSync( + process.execPath, + ["--require", preload, join(project, "scripts/install.js")], + { env, stdio: "pipe" }, + ); + const binary = join(project, "scripts/bt"); + assert.equal(readFileSync(binary, "utf8"), stub); + assert.equal(statSync(binary).mode & 0o777, 0o755); + assert.equal( + execFileSync(join(project, "bin/bt"), ["--version"], { + env, + encoding: "utf8", + }).trim(), + `bt ${version}`, + ); + }, + ); + + await t.test( + "missing archives fail before emitting a publishable wrapper", + () => { + const [target, spec] = Object.entries(targets)[0]; + rmSync(join(archives, `bt-${target}.${spec.archiveExt}`)); + const result = spawnSync(process.execPath, args, { encoding: "utf8" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Archive not found/); + assert.equal(existsSync(join(out, "bt/package.json")), false); + }, + ); + }, +); diff --git a/npm/tests/runtime.test.cjs b/npm/tests/runtime.test.cjs new file mode 100644 index 00000000..65f246a6 --- /dev/null +++ b/npm/tests/runtime.test.cjs @@ -0,0 +1,472 @@ +const assert = require("node:assert/strict"); +const { spawn } = require("node:child_process"); +const { EventEmitter } = require("node:events"); +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { test } = require("node:test"); +const vm = require("node:vm"); +const { gzipSync } = require("node:zlib"); +const { Worker } = require("node:worker_threads"); + +const packageDir = path.resolve(__dirname, "../bt"); +const helperSource = fs.readFileSync( + path.join(packageDir, "scripts/bt-helper.js"), + "utf8", +); +const installSource = fs.readFileSync( + path.join(packageDir, "scripts/install.js"), + "utf8", +); + +function loadHelper({ + platform = "linux", + arch = "x64", + libc = "glibc", + env = {}, + installed = [], + installedVersion = "1.2.3", + missingVersion = false, + fallback = false, +} = {}) { + const fakeRequire = (name) => { + if (name === "node:os") + return { platform: () => platform, arch: () => arch }; + if (name === "node:fs") return { existsSync: () => fallback }; + if (name === "../package.json") + return { + optionalDependencies: missingVersion + ? {} + : { "@braintrust/bt-linux-x64": "1.2.3" }, + }; + if ( + installed.some( + (binary) => + name === path.resolve("/resolved", binary, "../..", "package.json"), + ) + ) + return { version: installedVersion }; + return require(name); + }; + fakeRequire.resolve = (name) => { + if (installed.includes(name)) return `/resolved/${name}`; + throw Object.assign(new Error(`Cannot find module '${name}'`), { + code: "MODULE_NOT_FOUND", + }); + }; + const context = { + require: fakeRequire, + module: { exports: {} }, + __dirname: path.join(packageDir, "scripts"), + process: { + platform, + arch, + env, + report: { + getReport: () => ({ + header: libc === "glibc" ? { glibcVersionRuntime: "2.31" } : {}, + }), + }, + }, + }; + vm.runInNewContext(helperSource, context); + return context.module.exports; +} + +test("resolves all seven published platforms", () => { + const targets = require("../targets.json"); + for (const platform of Object.values(targets)) { + const pkg = require(`../platforms/bt-${platform}/package.json`); + const helper = loadHelper({ + platform: pkg.os[0], + arch: pkg.cpu[0], + libc: pkg.libc?.[0], + }); + const dist = helper.getDistributionForThisPlatform(); + assert.equal(dist.packageName, pkg.name); + assert.equal(dist.subpath, pkg.os[0] === "win32" ? "bin/bt.exe" : "bin/bt"); + } +}); + +test("binary override, optional dependency, then fallback precedence", () => { + const installed = ["@braintrust/bt-linux-x64/bin/bt"]; + assert.equal( + loadHelper({ + env: { BT_BINARY_PATH: "/custom/bt" }, + installed, + fallback: true, + }).getBinaryPath(), + "/custom/bt", + ); + assert.equal( + loadHelper({ installed, fallback: true }).getBinaryPath(), + "/resolved/@braintrust/bt-linux-x64/bin/bt", + ); + assert.equal( + loadHelper({ fallback: true }).getBinaryPath(), + path.join(packageDir, "scripts/bt"), + ); +}); + +test("rejects a hoisted binary with a different version and prefers the fallback", () => { + const options = { + installed: ["@braintrust/bt-linux-x64/bin/bt"], + installedVersion: "1.2.2", + }; + assert.throws( + () => loadHelper(options).getBinaryPath(), + /Expected @braintrust\/bt-linux-x64@1\.2\.3, but found 1\.2\.2.*Reinstall/, + ); + assert.equal( + loadHelper({ ...options, fallback: true }).getBinaryPath(), + path.join(packageDir, "scripts/bt"), + ); +}); + +test("ARM64 musl is unsupported even when a glibc binary is installed", () => { + const helper = loadHelper({ + arch: "arm64", + libc: "musl", + installed: ["@braintrust/bt-linux-arm64/bin/bt"], + }); + assert.equal(helper.getDistributionForThisPlatform().packageName, undefined); + assert.throws( + () => helper.getBinaryPath(), + /Unsupported operating system[\s\S]*Linux glibc on arm64 and x64; Linux musl on x64/, + ); +}); + +test("missing and wrong-platform dependencies give actionable errors", () => { + assert.throws( + () => loadHelper().getBinaryPath(), + /@braintrust\/bt.*optional dependencies/, + ); + assert.throws( + () => + loadHelper({ + installed: ["@braintrust/bt-darwin-arm64/bin/bt"], + }).getBinaryPath(), + /freshly install your dependencies on the target system/, + ); + assert.throws( + () => loadHelper({ platform: "freebsd" }).getBinaryPath(), + /Unsupported operating system/, + ); +}); + +// Exercise the unmodified lifecycle entry point with in-memory registry and +// filesystem stubs, without network access or user configuration changes. +async function installerWorker({ + env = {}, + installed = false, + installedVersion = "1.2.3", + arch = "x64", + libc = "glibc", + unsupported = false, + status = 200, + missingVersion = false, + invalidTar = false, + malformedTar = false, + truncatedTar = false, + networkError = false, + badChecksum = false, + missingChecksum = false, + writeError = false, +} = {}) { + const writes = []; + const messages = []; + const exits = []; + const requests = []; + const renames = []; + const removals = []; + const binary = Buffer.from("synthetic bt binary"); + const tar = Buffer.alloc(1536); + tar.write("package/bin/bt"); + tar.write(`${binary.length.toString(8).padStart(11, "0")}\0`, 124); + binary.copy(tar, 512); + if (malformedTar) tar.write("not-octal!!!\0", 124); + const { parentPort, workerData } = require("node:worker_threads"); + const finish = () => + parentPort.postMessage({ + writes, + messages: messages.join("\n"), + exits, + requests, + renames, + removals, + binary, + }); + const fakeRequire = (name) => { + if (name === "./bt-helper") + return { + ...loadHelper({ + platform: unsupported ? "freebsd" : "linux", + arch, + libc, + installed: installed ? ["@braintrust/bt-linux-x64/bin/bt"] : [], + installedVersion, + missingVersion, + }), + getFallbackBinaryPath: () => "/fallback/bt", + }; + if (name === "../package.json") + return { + optionalDependencies: missingVersion + ? {} + : { "@braintrust/bt-linux-x64": "1.2.3" }, + }; + if (name === "../checksums.json") + return missingChecksum + ? {} + : { + "@braintrust/bt-linux-x64": require("node:crypto") + .createHash("sha256") + .update(badChecksum ? "different binary" : binary) + .digest("hex"), + }; + if (name === "node:fs") + return { + mkdirSync: () => {}, + writeFileSync: (file, content, options) => { + if (writeError) throw new Error("disk full"); + writes.push({ file, content, mode: options.mode }); + }, + renameSync: (from, to) => renames.push({ from, to }), + rmSync: (file) => removals.push(file), + }; + if (name === "node:https") + return { + get: (url, callback) => { + requests.push(url); + const request = new EventEmitter(); + queueMicrotask(() => { + if (networkError) + return request.emit("error", new Error("offline")); + const response = new EventEmitter(); + response.statusCode = status; + response.headers = {}; + response.resume = () => {}; + callback(response); + response.emit( + "data", + gzipSync( + invalidTar + ? Buffer.alloc(512) + : truncatedTar + ? tar.subarray(0, 512) + : tar, + ), + ); + response.emit("end"); + }); + return request; + }, + }; + return require(name); + }; + vm.runInNewContext(workerData.source, { + require: fakeRequire, + Buffer, + console: { + log: (message) => messages.push(message), + error: (message) => messages.push(message), + }, + process: { + env, + platform: "linux", + arch, + pid: 123, + exit: (code) => { + exits.push(code); + finish(); + process.exit(code); + }, + }, + }); + await new Promise((resolve) => setImmediate(resolve)); + finish(); +} + +function runInstaller(options = {}) { + const worker = new Worker( + ` + const { EventEmitter } = require("node:events"); + const vm = require("node:vm"); + const path = require("node:path"); + const { gzipSync } = require("node:zlib"); + const { workerData } = require("node:worker_threads"); + const packageDir = workerData.packageDir; + const helperSource = workerData.helperSource; + const loadHelper = ${loadHelper.toString()}; + (${installerWorker.toString()})(workerData.options); + `, + { + eval: true, + workerData: { source: installSource, helperSource, packageDir, options }, + }, + ); + return new Promise((resolve, reject) => { + let result; + worker.on("message", (message) => { + result = message; + }); + worker.on("error", reject); + worker.on("exit", (code) => { + if (!result) reject(new Error(`Installer worker exited with ${code}`)); + else resolve({ ...result, exitCode: code }); + }); + }); +} + +test("postinstall skips downloads for installed dependencies or explicit overrides", async () => { + for (const options of [ + { installed: true }, + { env: { BT_SKIP_DOWNLOAD: "1" } }, + { env: { BT_SKIP_DOWNLOAD: "1" }, unsupported: true }, + { env: { BT_BINARY_PATH: "/custom/bt" } }, + { env: { BT_BINARY_PATH: "/custom/bt" }, unsupported: true }, + ]) { + const result = await runInstaller(options); + assert.equal(result.requests.length, 0); + assert.deepEqual(result.exits, [0]); + assert.equal(result.exitCode, 0); + } +}); + +test("postinstall downloads the exact pinned binary and makes it executable", async () => { + const result = await runInstaller(); + assert.equal(result.exitCode, 0); + assert.deepEqual(result.requests, [ + "https://registry.npmjs.org/@braintrust/bt-linux-x64/-/bt-linux-x64-1.2.3.tgz", + ]); + assert.deepEqual(result.writes, [ + { file: "/fallback/bt.123.tmp", content: result.binary, mode: 0o755 }, + ]); + assert.deepEqual(result.renames, [ + { from: "/fallback/bt.123.tmp", to: "/fallback/bt" }, + ]); + assert.deepEqual(result.removals, ["/fallback/bt.123.tmp"]); +}); + +test("postinstall downloads the pinned version when an older hoisted binary exists", async () => { + const result = await runInstaller({ + installed: true, + installedVersion: "1.2.2", + }); + assert.equal(result.exitCode, 0); + assert.deepEqual(result.requests, [ + "https://registry.npmjs.org/@braintrust/bt-linux-x64/-/bt-linux-x64-1.2.3.tgz", + ]); + assert.equal(result.writes.length, 1); + assert.equal(result.renames.length, 1); + assert.match(result.messages, /Expected .*@1\.2\.3, but found 1\.2\.2/); +}); + +test("postinstall fails without downloading a glibc binary on ARM64 musl", async () => { + const result = await runInstaller({ arch: "arm64", libc: "musl" }); + assert.deepEqual(result.requests, []); + assert.deepEqual(result.writes, []); + assert.deepEqual(result.exits, [1]); + assert.equal(result.exitCode, 1); + assert.match(result.messages, /no prebuilt binary available for linux-arm64/); +}); + +test("postinstall fails for unsupported platforms and download failures", async () => { + for (const [options, message] of [ + [{ unsupported: true }, /no prebuilt binary/], + [{ missingVersion: true }, /cannot determine which version/], + [{ status: 503 }, /status code 503/], + [{ networkError: true }, /offline/], + [{ invalidTar: true }, /could not find/], + [{ malformedTar: true }, /Invalid or truncated tar entry/], + [{ truncatedTar: true }, /Invalid or truncated tar entry/], + [{ badChecksum: true }, /Checksum validation failed/], + [{ missingChecksum: true }, /No release checksum found/], + [{ writeError: true }, /disk full/], + ]) { + const result = await runInstaller(options); + assert.deepEqual(result.exits, [1]); + assert.equal(result.exitCode, 1); + assert.equal(result.writes.length, 0); + assert.equal(result.renames.length, 0); + if (options.writeError) + assert.deepEqual(result.removals, ["/fallback/bt.123.tmp"]); + assert.match(result.messages, message); + assert.doesNotMatch(result.messages, /SDK is unaffected/); + } +}); + +function launch(args, options = {}) { + return spawn(process.execPath, [path.join(packageDir, "bin/bt"), ...args], { + env: { ...process.env, BT_BINARY_PATH: process.execPath }, + ...options, + }); +} + +test("launcher forwards arguments, stdin, stdout, stderr, and exit codes", async () => { + const child = launch([ + "-e", + 'process.stdin.on("data", data => { process.stdout.write(JSON.stringify(process.argv.slice(1))); process.stdout.write(data); process.stderr.write("diagnostic"); process.exitCode = 7; });', + "--", + "argument with spaces", + "--flag", + ]); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const completed = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("close", resolve); + }); + child.stdin.end("input"); + assert.equal(await completed, 7); + assert.equal(stdout, '["argument with spaces","--flag"]input'); + assert.equal(stderr, "diagnostic"); +}); + +test("launcher reports spawn failures", async () => { + const child = launch([], { + env: { + ...process.env, + BT_BINARY_PATH: path.join(os.tmpdir(), "bt-test-nonexistent", "bt"), + }, + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + assert.equal(await new Promise((resolve) => child.on("close", resolve)), 1); + assert.match(stderr, /ENOENT/); +}); + +for (const signal of ["SIGINT", "SIGTERM"]) { + test( + `launcher forwards and re-raises ${signal}`, + { skip: process.platform === "win32", timeout: 10000 }, + async (t) => { + const child = launch([ + "-e", + `process.on("${signal}", () => { process.stderr.write("forwarded"); process.removeAllListeners("${signal}"); process.kill(process.pid, "${signal}"); }); process.stdout.write("ready"); setInterval(() => {}, 1000);`, + ]); + t.after(() => child.kill("SIGKILL")); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + const completed = new Promise((resolve) => + child.on("close", (code, receivedSignal) => + resolve({ code, signal: receivedSignal }), + ), + ); + await new Promise((resolve) => child.stdout.once("data", resolve)); + child.kill(signal); + assert.deepEqual(await completed, { code: null, signal }); + assert.equal(stderr, "forwarded"); + }, + ); +} From af10f50e23e34894366c4acc9e33d61072278b79 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:37:00 +0000 Subject: [PATCH 2/6] Update PR #378 --- .github/workflows/release-canary.yml | 19 +++++++++++++++++-- .github/workflows/release.yml | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-canary.yml b/.github/workflows/release-canary.yml index 03dd8ce0..94425629 100644 --- a/.github/workflows/release-canary.yml +++ b/.github/workflows/release-canary.yml @@ -50,7 +50,11 @@ jobs: - name: Install dist shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh" + run: | + curl --proto '=https' --tlsv1.2 -LsSf --retry 3 \ + https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh \ + --output "$RUNNER_TEMP/cargo-dist-installer.sh" + sh "$RUNNER_TEMP/cargo-dist-installer.sh" - name: Cache dist uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -149,7 +153,18 @@ jobs: chmod +x /tmp/sha256sum echo /tmp >> "$GITHUB_PATH" - - name: Install dist + - name: Install dist (Unix) + if: runner.os != 'Windows' + shell: bash + run: | + curl --proto '=https' --tlsv1.2 -LsSf --retry 3 \ + https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh \ + --output "$RUNNER_TEMP/cargo-dist-installer.sh" + sh "$RUNNER_TEMP/cargo-dist-installer.sh" + + - name: Install dist (Windows) + if: runner.os == 'Windows' + shell: pwsh run: ${{ matrix.install_dist.run }} - name: Fetch local artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4847d73c..e2bae097 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,11 @@ jobs: - name: Install dist if: steps.resolve-tag.outputs.should_release == 'true' shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh | sh" + run: | + curl --proto '=https' --tlsv1.2 -LsSf --retry 3 \ + https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh \ + --output "$RUNNER_TEMP/cargo-dist-installer.sh" + sh "$RUNNER_TEMP/cargo-dist-installer.sh" - name: Cache dist if: steps.resolve-tag.outputs.should_release == 'true' @@ -187,7 +191,18 @@ jobs: chmod +x /tmp/sha256sum echo /tmp >> "$GITHUB_PATH" - - name: Install dist + - name: Install dist (Unix) + if: runner.os != 'Windows' + shell: bash + run: | + curl --proto '=https' --tlsv1.2 -LsSf --retry 3 \ + https://github.com/axodotdev/cargo-dist/releases/download/v0.31.0/cargo-dist-installer.sh \ + --output "$RUNNER_TEMP/cargo-dist-installer.sh" + sh "$RUNNER_TEMP/cargo-dist-installer.sh" + + - name: Install dist (Windows) + if: runner.os == 'Windows' + shell: pwsh run: ${{ matrix.install_dist.run }} - name: Fetch local artifacts From 59dbdf0c4a91f8c76bf9b7ecb2046d3b36bb19a6 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Thu, 24 Sep 2026 21:51:16 +0000 Subject: [PATCH 3/6] use shared workflows --- .github/workflows/release.yml | 375 +++++++++++++++--------- CONTRIBUTING.md | 29 -- README.md | 50 +++- npm/scripts/build-platform-packages.mjs | 67 ++++- npm/tests/packages.test.mjs | 74 ++++- 5 files changed, 402 insertions(+), 193 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2bae097..d7b324ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,13 @@ jobs: tag: ${{ steps.resolve-tag.outputs.tag }} tag-flag: ${{ steps.resolve-tag.outputs.tag_flag }} release-version: ${{ steps.resolve-tag.outputs.release_version }} + release-sha: ${{ steps.resolve-tag.outputs.release_sha }} should-release: ${{ steps.resolve-tag.outputs.should_release }} + approval-manifest: ${{ steps.approval-manifest.outputs.manifest }} + branch: ${{ steps.configure.outputs.branch }} + on-release-branch: ${{ steps.configure.outputs.on_release_branch }} + commit-message: ${{ steps.configure.outputs.commit_message }} + notes: ${{ steps.configure.outputs.notes }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -47,13 +53,15 @@ jobs: - id: resolve-tag name: Resolve release tag shell: bash + env: + REQUESTED_TAG: ${{ inputs.tag }} run: | set -euo pipefail should_release=true if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - tag="${{ inputs.tag }}" + tag="$REQUESTED_TAG" elif [[ "${{ github.ref }}" == refs/tags/* ]]; then tag="${{ github.ref_name }}" else @@ -89,6 +97,7 @@ jobs: echo "tag=$tag" >> "$GITHUB_OUTPUT" echo "tag_flag=--tag=$tag" >> "$GITHUB_OUTPUT" echo "release_version=${tag#v}" >> "$GITHUB_OUTPUT" + echo "release_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" echo "should_release=$should_release" >> "$GITHUB_OUTPUT" printf '%s\n' "$tag" > release-tag.txt @@ -98,6 +107,38 @@ jobs: name: release-metadata path: release-tag.txt + - name: Configure release metadata + id: configure + if: steps.resolve-tag.outputs.should_release == 'true' + uses: braintrustdata/sdk-actions/actions/release/lang/js/configure@337c5475631d724441403f5a4d8ad0b32ce12e29 + with: + sha: ${{ steps.resolve-tag.outputs.release_sha }} + version: ${{ steps.resolve-tag.outputs.release_version }} + working_directory: npm/bt + npm_package_name: "@braintrust/bt" + package_label: Braintrust CLI + + - name: Prepare approval manifest + id: approval-manifest + if: steps.resolve-tag.outputs.should_release == 'true' + env: + RELEASE_PACKAGE: ${{ steps.configure.outputs.package }} + RELEASE_VERSION: ${{ steps.resolve-tag.outputs.release_version }} + run: | + # One native release plus all npm distributions. Only the wrapper's + # registry state is known here; the publisher checks each tarball again. + manifest="$(jq -cn --argjson release "$RELEASE_PACKAGE" --arg version "$RELEASE_VERSION" \ + --slurpfile wrapper npm/bt/package.json ' + {packages: [ + ($release | del(.registries)), + (($wrapper[0].optionalDependencies | keys[]) as $name | { + label: $name, tag: $version, release_body: "Native binary distribution on npm.", + registries: {npm: {name: $name, version: $version, channel: "latest"}} + }), + ($release + {label: $wrapper[0].name, notes: "", release_body: "Standalone CLI on npm."}) + ]}')" + echo "manifest=$manifest" >> "$GITHUB_OUTPUT" + - name: Configure git auth for private dependencies if: steps.resolve-tag.outputs.should_release == 'true' shell: bash @@ -140,18 +181,38 @@ jobs: name: artifacts-plan-dist-manifest path: plan-dist-manifest.json + request-approval: + needs: plan + if: needs.plan.outputs.should-release == 'true' + runs-on: ubuntu-22.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Request stable release approval + uses: braintrustdata/sdk-actions/actions/release/request-approval@337c5475631d724441403f5a4d8ad0b32ce12e29 + with: + packages: ${{ needs.plan.outputs.approval-manifest }} + title: Braintrust CLI stable release + sha: ${{ needs.plan.outputs.release-sha }} + branch: ${{ needs.plan.outputs.branch }} + on_release_branch: ${{ needs.plan.outputs.on-release-branch }} + commit_message: ${{ needs.plan.outputs.commit-message }} + slack_token: ${{ secrets.SLACK_BOT_TOKEN }} + slack_channel: C0ABHT0SWA2 + build-local-artifacts: name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) needs: - plan + - request-approval if: ${{ needs.plan.outputs.should-release == 'true' && fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null }} strategy: fail-fast: false matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} runs-on: ${{ matrix.runner }} container: ${{ matrix.container && matrix.container.image || null }} - # OIDC token subject must match the AAD federated credential, which is - # scoped to the `release` environment. + # Required reviewers on `release` gate the entire native + npm release. + # Windows signing's AAD federated credential also requires this environment. environment: release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -163,6 +224,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ needs.plan.outputs.release-sha }} persist-credentials: false submodules: recursive @@ -288,6 +350,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ needs.plan.outputs.release-sha }} persist-credentials: false submodules: recursive @@ -340,7 +403,7 @@ jobs: - plan - build-local-artifacts - build-global-artifacts - if: ${{ always() && needs.plan.outputs.should-release == 'true' && needs.plan.result == 'success' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + if: needs.plan.outputs.should-release == 'true' runs-on: ubuntu-22.04 env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -349,6 +412,7 @@ jobs: steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ needs.plan.outputs.release-sha }} persist-credentials: false submodules: recursive @@ -390,91 +454,170 @@ jobs: name: artifacts-dist-manifest path: dist-manifest.json - announce: + build-npm: needs: - plan - host - if: ${{ always() && needs.host.result == 'success' }} runs-on: ubuntu-22.04 - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + permissions: + contents: read + id-token: write + attestations: write + outputs: + manifest: ${{ steps.packages.outputs.manifest }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: + ref: ${{ needs.plan.outputs.release-sha }} persist-credentials: false - submodules: recursive - fetch-depth: 0 - fetch-tags: true - - name: Download GitHub artifacts + - name: Download native release artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - pattern: artifacts-* - path: artifacts + pattern: artifacts-build-local-* + path: archives merge-multiple: true - - name: Cleanup - run: rm -f artifacts/*-dist-manifest.json + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + - name: Use npm 11.12.1 + run: npm install -g npm@11.12.1 - - name: Create GitHub release + - name: Build npm packages + id: packages env: - PRERELEASE_FLAG: "${{ fromJson(needs.host.outputs.val).announcement_is_prerelease && '--prerelease' || '' }}" - ANNOUNCEMENT_TITLE: "${{ fromJson(needs.host.outputs.val).announcement_title }}" - ANNOUNCEMENT_BODY: "${{ fromJson(needs.host.outputs.val).announcement_github_body }}" - RELEASE_COMMIT: "${{ github.sha }}" - RELEASE_TAG: "${{ needs.plan.outputs.tag }}" - REPO: "${{ github.repository }}" - shell: bash + RELEASE_SHA: ${{ needs.plan.outputs.release-sha }} + RELEASE_TAG: ${{ needs.plan.outputs.tag }} + RELEASE_TITLE: ${{ fromJSON(needs.host.outputs.val).announcement_title }} + INSTALL_NOTES: ${{ fromJSON(needs.host.outputs.val).announcement_github_body }} + CHANGELOG: ${{ needs.plan.outputs.notes }} run: | - set -euo pipefail + node npm/scripts/build-platform-packages.mjs \ + --version "${{ needs.plan.outputs.release-version }}" \ + --archives-dir archives \ + --out-dir npm/dist + # All npm distributions belong to the same native CLI release/tag. + jq --arg sha "$RELEASE_SHA" --arg tag "$RELEASE_TAG" \ + --arg title "$RELEASE_TITLE" --arg install "$INSTALL_NOTES" \ + --arg notes "$(printf '%s' "$CHANGELOG" | base64 --decode)" \ + '.commit = $sha | .packages |= map(. + { + tag: $tag, release_title: $title, release_body: ($install + "\n\n" + $notes) + })' npm/dist/artifacts/release-manifest.json > "$RUNNER_TEMP/release-manifest.json" + mv "$RUNNER_TEMP/release-manifest.json" npm/dist/artifacts/release-manifest.json + echo "manifest=$(jq -c . npm/dist/artifacts/release-manifest.json)" >> "$GITHUB_OUTPUT" + + - name: Attest npm build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: npm/dist/artifacts/*.tgz - echo "$ANNOUNCEMENT_BODY" > "$RUNNER_TEMP/notes.txt" - - # Append the commits included in this release (previous stable tag -> this release). - # - # Resolve the commit this release actually points at. On a manual - # workflow_dispatch the tag may already exist and point at a different - # commit than the workflow's github.sha, so prefer the tag when it - # resolves; otherwise fall back to the release commit (the normal push - # flow creates the tag at this commit in the release step below). - release_ref="$RELEASE_COMMIT" - if git rev-parse -q --verify "refs/tags/${RELEASE_TAG}^{commit}" >/dev/null 2>&1; then - release_ref="refs/tags/${RELEASE_TAG}^{commit}" - fi + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: npm-packages + path: npm/dist/artifacts/ + if-no-files-found: error + + attest-npm-sboms: + needs: build-npm + runs-on: ubuntu-22.04 + permissions: + contents: read + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + package: ${{ fromJSON(needs.build-npm.outputs.manifest).packages }} + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-packages + path: release-artifacts - # Find the most recent stable v.. tag that precedes - # this release in its own ancestry (--merged), so re-publishing an older - # or backport release never picks an unrelated newer tag as the previous - # one. canary-* and other non-release tags are ignored by the glob. - previous_tag="$(git tag --list 'v[0-9]*.[0-9]*.[0-9]*' --merged "$release_ref" --sort=-version:refname \ - | grep -vFx "$RELEASE_TAG" | head -n1 || true)" + - name: Attest npm package SBOM + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-path: release-artifacts/${{ matrix.package.tarball_asset }} + sbom-path: release-artifacts/${{ matrix.package.sbom_asset }} + + publish-release: + needs: + - plan + - build-npm + - attest-npm-sboms + runs-on: ubuntu-22.04 + timeout-minutes: 30 + # Approval happened on `release` before the builds. Keep the npm OIDC identity. + environment: npm-publish + permissions: + contents: write + id-token: write + attestations: read + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ needs.plan.outputs.release-sha }} + persist-credentials: false + fetch-depth: 0 + + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: npm-packages + path: release-artifacts + + - name: Download native release artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: artifacts-* + path: native-artifacts + merge-multiple: true - if [ -n "$previous_tag" ]; then - commit_range="${previous_tag}..${release_ref}" + - name: Ensure release tag points to the approved commit + env: + GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ needs.plan.outputs.release-sha }} + RELEASE_TAG: ${{ needs.plan.outputs.tag }} + run: | + set -euo pipefail + if git show-ref --verify --quiet "refs/tags/$RELEASE_TAG"; then + test "$(git rev-parse "refs/tags/$RELEASE_TAG^{commit}")" = "$RELEASE_SHA" else - # First release (no earlier tag in ancestry): include the whole history. - commit_range="$release_ref" + gh api "repos/$GITHUB_REPOSITORY/git/refs" --method POST \ + -f ref="refs/tags/$RELEASE_TAG" -f sha="$RELEASE_SHA" fi - commits="$(git log --no-merges --pretty=format:'- %s (%h)' "$commit_range" || true)" - if [ -n "$commits" ]; then - { - printf '\n## Commits\n\n' - printf '%s\n' "$commits" - if [ -n "$previous_tag" ]; then - printf '\n**Full Changelog**: https://github.com/%s/compare/%s...%s\n' \ - "$REPO" "$previous_tag" "$RELEASE_TAG" - fi - } >> "$RUNNER_TEMP/notes.txt" - fi + - name: Publish platform packages and standalone CLI + uses: braintrustdata/sdk-actions/actions/release/lang/js/publish-npm-tarballs@337c5475631d724441403f5a4d8ad0b32ce12e29 + with: + manifest: release-artifacts/release-manifest.json + tarballs: release-artifacts/*.tgz + npm_version: "11.12.1" + slack_token: ${{ secrets.SLACK_BOT_TOKEN }} + slack_channel: C0ABHT0SWA2 + + - name: Create GitHub release + uses: braintrustdata/sdk-actions/actions/release/create-package-github-releases@337c5475631d724441403f5a4d8ad0b32ce12e29 + with: + manifest: release-artifacts/release-manifest.json - gh release create "$RELEASE_TAG" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + - name: Attach native artifacts and all npm SBOMs + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ needs.plan.outputs.tag }} + run: | + # The shared action skips existing releases, including packages sharing + # one tag. Upload every SBOM here so reruns complete partial releases. + rm -f native-artifacts/*-dist-manifest.json + gh release upload "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --clobber \ + native-artifacts/* release-artifacts/*.sbom.json smoke-install-unix: needs: - plan - - announce - if: ${{ needs.announce.result == 'success' }} + - publish-release + if: ${{ needs.publish-release.result == 'success' }} runs-on: ubuntu-22.04 env: RELEASE_TAG: ${{ needs.plan.outputs.tag }} @@ -501,9 +644,9 @@ jobs: smoke-install-macos: needs: - plan - - announce - if: ${{ needs.announce.result == 'success' && vars.BT_ENABLE_MACOS_SMOKE == '1' }} - runs-on: macos-13 + - publish-release + if: ${{ needs.publish-release.result == 'success' && vars.BT_ENABLE_MACOS_SMOKE == '1' }} + runs-on: macos-latest env: RELEASE_TAG: ${{ needs.plan.outputs.tag }} REPO: ${{ github.repository }} @@ -524,8 +667,8 @@ jobs: smoke-install-windows: needs: - plan - - announce - if: ${{ needs.announce.result == 'success' }} + - publish-release + if: ${{ needs.publish-release.result == 'success' }} runs-on: windows-2022 env: RELEASE_TAG: ${{ needs.plan.outputs.tag }} @@ -549,78 +692,10 @@ jobs: $env:PATH = "$binDir;$env:PATH" & $btExe update --check --channel stable - publish-npm: - needs: - - plan - - announce - if: ${{ needs.announce.result == 'success' }} - runs-on: ubuntu-22.04 - environment: npm-publish - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - - name: Download release archives - env: - TAG: ${{ needs.plan.outputs.tag }} - shell: bash - run: | - set -euo pipefail - mkdir -p archives - gh release download "$TAG" \ - --repo "${{ github.repository }}" \ - --pattern 'bt-*.tar.gz' \ - --pattern 'bt-*.zip' \ - --dir archives - ls -la archives - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "24" - registry-url: "https://registry.npmjs.org" - - name: Use npm 11.12.1 - run: npm install -g npm@11.12.1 - - - name: Build npm packages - run: | - node npm/scripts/build-platform-packages.mjs \ - --version "${{ needs.plan.outputs.release-version }}" \ - --archives-dir archives \ - --out-dir npm/dist - - - name: Publish platform packages and standalone CLI - shell: bash - run: | - set -euo pipefail - shopt -s nullglob - package_dirs=(npm/dist/bt-*) - if [ "${#package_dirs[@]}" -eq 0 ]; then - echo "No npm platform packages found in npm/dist" >&2 - exit 1 - fi - # Publish the wrapper last so every exact optional dependency exists. - package_dirs+=(npm/dist/bt) - for dir in "${package_dirs[@]}"; do - name="$(node -p "require('./$dir/package.json').name")" - version="$(node -p "require('./$dir/package.json').version")" - # Skip already-published versions so a partial publish can be re-run. - # Test output, not exit code: npm view is empty for a missing version. - published="$(npm view "$name@$version" version 2>/dev/null || true)" - if [ -n "$published" ]; then - echo "Skipping $name@$version (already published)" - continue - fi - echo "Publishing $name@$version" - (cd "$dir" && npm publish --access public --tag latest) - done - smoke-npm: needs: - plan - - publish-npm + - publish-release runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -641,3 +716,31 @@ jobs: npm install --global --ignore-scripts "@braintrust/bt@${RELEASE_VERSION}" test "$(bt --version)" = "bt ${RELEASE_VERSION}" test "$(npx --yes "@braintrust/bt@${RELEASE_VERSION}" --version)" = "bt ${RELEASE_VERSION}" + + notify-failure: + needs: + - plan + - request-approval + - build-local-artifacts + - build-global-artifacts + - host + - build-npm + - attest-npm-sboms + - publish-release + - smoke-install-unix + - smoke-install-macos + - smoke-install-windows + - smoke-npm + if: ${{ failure() }} + runs-on: ubuntu-22.04 + timeout-minutes: 5 + permissions: {} + steps: + - name: Notify release failure + uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: C0ABHT0SWA2 + text: "Braintrust CLI stable release failed. <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View run>" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a9542028..3b3c2333 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -140,35 +140,6 @@ Notes: - The workflow publishes an immutable tag: `canary--`. - It also updates a moving branch tag: `canary-` when the run is for the latest commit on that branch. -## npm Releases - -Stable releases publish seven platform binary packages, then the standalone `@braintrust/bt` package, from the `publish-npm` job in `release.yml`. -The wrapper and its optional dependencies use the same exact CLI release version. -Installation follows the [optional dependencies plus postinstall fallback pattern](https://blog.sentry.io/publishing-binaries-on-npm/). -The builder also embeds SHA-256 checksums of the release binaries in the wrapper, following Sentry CLI's checksum approach; fallback downloads must match before they are installed. -Existing published versions are skipped so a partially completed publishing job can be retried. -Canary workflows do not publish npm packages. - -Package manifests are checked in under `npm/bt` and `npm/platforms`, including the wrapper's explicit optional dependencies and each platform's OS, CPU, and libc constraints. -Their `0.0.0` versions are placeholders; the release script stamps the release version into all manifests and dependency pins, copies the binaries, and calculates checksums. -`npm/targets.json` only maps Rust targets to platform package directories. -Publish the prepared packages under `npm/dist`, not the source directories. -To prepare packages from downloaded release archives: - -```bash -node npm/scripts/build-platform-packages.mjs --version --archives-dir -``` - -Before the first standalone release, an npm organization maintainer must publish the generated `@braintrust/bt` package with their authenticated npm account after its matching platform packages are available. -For this one-time local bootstrap, run `npm publish ./npm/dist/bt --access public --provenance=false`; subsequent CI publishes generate provenance through OIDC. -Then configure its npm trusted publisher with organization `braintrustdata`, repository `bt`, workflow filename `release.yml`, and environment `npm-publish`, allowing direct publishing. -Subsequent releases use the existing workflow's OIDC authentication without an npm publishing token. -Make the standalone package available before releasing the SDK change that removes its bundled CLI. - -Run packaging tests with `node --test npm/tests/*.test.cjs npm/tests/*.test.mjs`. -The archive builder tests require `tar`, `unzip`, and Python 3 and run on Linux in CI. -Runtime tests also run on macOS and Windows. - ## Windows Release Signing Release workflows Authenticode-sign Windows artifacts via [Azure Trusted Signing](https://learn.microsoft.com/azure/trusted-signing/) (the `sign-windows-artifacts` composite action). Signing runs when these GitHub Actions repository secrets are configured: diff --git a/README.md b/README.md index 3e24c343..66aa17f0 100644 --- a/README.md +++ b/README.md @@ -6,19 +6,6 @@ ## Install -### npm - -```bash -npm install -g @braintrust/bt -bt --version -``` - -For a project-local installation, use `npm install --save-dev @braintrust/bt` and run `npx bt`. -To run without adding a project dependency, use `npx @braintrust/bt --help`. -Append `@` to the package name to install a specific stable CLI release. -The CLI is installed separately from the `braintrust` JavaScript SDK. -See the [npm package README](npm/bt/README.md) for supported platforms and fallback download behavior. - ### Unix (macOS / Linux) ```bash @@ -86,6 +73,43 @@ The SHA1 is the commit hash while the (optional) SHA256 of `mise.toml` are hashe Non-`main` branch builds are available as GitHub Actions run artifacts (download from the workflow run page or with `gh run download`). They are not published as GitHub Releases. +### npm, Yarn, and pnpm + +Install the CLI in your project using your package manager: + +```bash +# npm +npm install --save-dev @braintrust/bt + +# Yarn +yarn add --dev @braintrust/bt + +# pnpm +pnpm add --save-dev @braintrust/bt +``` + +Once installed, you can invoke `bt` directly from scripts in your `package.json`: + +```json +{ + "scripts": { + "bt": "bt" + } +} +``` + +For example, run `npm run bt -- --help`, `yarn run bt --help`, or `pnpm run bt --help`. + +To install globally with npm: + +```bash +npm install -g @braintrust/bt +bt --version +``` + +To run without adding a project dependency, use `npx @braintrust/bt --help`. +See the [npm package README](npm/bt/README.md) for supported platforms and fallback download behavior. + ## Verify ```bash diff --git a/npm/scripts/build-platform-packages.mjs b/npm/scripts/build-platform-packages.mjs index 048be6e1..9d1f92f3 100644 --- a/npm/scripts/build-platform-packages.mjs +++ b/npm/scripts/build-platform-packages.mjs @@ -1,13 +1,14 @@ #!/usr/bin/env node -// Prepares checked-in npm packages using cargo-dist release archives. +// Prepares checked-in npm packages using cargo-dist release archives, then +// packs tarballs and npm dependency SBOMs for sdk-actions to attest and publish. // // --version version to stamp into every package.json (required) // --archives-dir directory containing cargo-dist archives // (bt-.tar.gz / bt-.zip), required // --out-dir directory to write packages into (default: npm/dist) // -// Emits /bt-/ (one per target) and /bt/, each ready -// to `npm publish`. The @braintrust/bt package exposes the `bt` command. +// Emits /bt-/ (one per target), /bt/, and +// /artifacts/ containing the tarballs, SBOMs, and release manifest. import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; @@ -18,6 +19,7 @@ import { mkdirSync, readFileSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { dirname, join, resolve } from "node:path"; @@ -137,5 +139,62 @@ writeFileSync( ); console.log(`Built @braintrust/bt -> ${wrapperOut}`); +// Make every platform visible to npm sbom, including platforms other than the +// build host. These exact dependencies aren't published yet, so use the local +// packages without downloading dependencies or running installation scripts. +const wrapperModules = join(wrapperOut, "node_modules"); +mkdirSync(join(wrapperModules, "@braintrust"), { recursive: true }); +const packageDirs = Object.values(targets).map((platform) => `bt-${platform}`); +const artifactsDir = join(outDir, "artifacts"); +mkdirSync(artifactsDir); +const packages = []; +try { + for (const dir of packageDirs) { + symlinkSync( + join(outDir, dir), + join(wrapperModules, "@braintrust", dir), + "dir", + ); + } + + // The shared publisher consumes this order: dependencies before the wrapper. + for (const dir of [...packageDirs, "bt"]) { + const cwd = join(outDir, dir); + const [packed] = JSON.parse( + execFileSync( + "npm", + [ + "pack", + "--json", + "--ignore-scripts", + "--pack-destination", + artifactsDir, + ], + { cwd, encoding: "utf8" }, + ), + ); + const sbomAsset = packed.filename.replace(/\.tgz$/, ".sbom.json"); + const sbom = execFileSync( + "npm", + ["sbom", "--sbom-format=cyclonedx", "--omit=dev"], + { cwd, encoding: "utf8" }, + ); + writeFileSync(join(artifactsDir, sbomAsset), sbom); + packages.push({ + name: packed.name, + version: packed.version, + tarball_asset: packed.filename, + sbom_asset: sbomAsset, + }); + } +} finally { + rmSync(wrapperModules, { recursive: true, force: true }); +} + +writeFileSync( + join(artifactsDir, "release-manifest.json"), + JSON.stringify({ packages }, null, 2) + "\n", +); + const expected = Object.keys(targets).length + 1; -console.log(`\nAll ${expected} packages written to ${outDir}`); +console.log(`\nAll ${expected} packages packed in ${artifactsDir}`); diff --git a/npm/tests/packages.test.mjs b/npm/tests/packages.test.mjs index 123a5148..87891370 100644 --- a/npm/tests/packages.test.mjs +++ b/npm/tests/packages.test.mjs @@ -79,10 +79,67 @@ test( const wrapper = JSON.parse( readFileSync(join(out, "bt/package.json"), "utf8"), ); + const artifacts = join(out, "artifacts"); + const manifest = JSON.parse( + readFileSync(join(artifacts, "release-manifest.json"), "utf8"), + ); const env = { ...process.env, npm_config_cache: join(root, "cache") }; delete env.BT_BINARY_PATH; delete env.BT_SKIP_DOWNLOAD; + await t.test( + "release manifest orders packed platforms before the wrapper with complete npm SBOMs", + () => { + const platformNames = Object.keys(wrapper.optionalDependencies); + assert.deepEqual( + manifest.packages.map((pkg) => pkg.name), + [...platformNames, wrapper.name], + ); + for (const pkg of manifest.packages) { + assert.equal(pkg.version, version); + const tarball = join(artifacts, pkg.tarball_asset); + const packed = JSON.parse( + execFileSync("tar", ["-xOf", tarball, "package/package.json"], { + encoding: "utf8", + }), + ); + assert.equal(packed.name, pkg.name); + assert.equal(packed.version, version); + const files = execFileSync("tar", ["-tf", tarball], { + encoding: "utf8", + }); + assert.doesNotMatch(files, /node_modules/); + const sbom = JSON.parse( + readFileSync(join(artifacts, pkg.sbom_asset), "utf8"), + ); + assert.equal(sbom.bomFormat, "CycloneDX"); + assert.equal( + sbom.metadata.component["bom-ref"], + `${pkg.name}@${version}`, + ); + assert.equal(sbom.metadata.component.version, version); + const expectedDeps = pkg.name === wrapper.name ? platformNames : []; + assert.deepEqual( + sbom.components.map((component) => component["bom-ref"]).sort(), + expectedDeps.map((name) => `${name}@${version}`).sort(), + ); + for (const component of sbom.components) { + assert.equal(component.version, version); + } + assert.deepEqual( + sbom.dependencies + .find( + (dependency) => + dependency.ref === sbom.metadata.component["bom-ref"], + ) + .dependsOn.sort(), + expectedDeps.map((name) => `${name}@${version}`).sort(), + ); + } + assert.equal(existsSync(join(out, "bt/node_modules")), false); + }, + ); + await t.test( "wrapper pins every platform and includes only the runtime files", () => { @@ -157,17 +214,12 @@ test( const hostName = helper .getDistributionForThisPlatform() .packageName.split("/")[1]; - const tarballs = []; - for (const name of ["bt", hostName]) { - const [packed] = JSON.parse( - execFileSync( - "npm", - ["pack", "--json", "--ignore-scripts", "--pack-destination", root], - { cwd: join(out, name), env, encoding: "utf8" }, - ), - ); - tarballs.push(join(root, packed.filename)); - } + const tarballs = ["@braintrust/bt", `@braintrust/${hostName}`].map((name) => + join( + artifacts, + manifest.packages.find((pkg) => pkg.name === name).tarball_asset, + ), + ); await t.test( "local and global packed installs work with lifecycle scripts disabled", From 8e5f2670d02bee1a024501996aca7260e6d702f9 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:15:32 +0000 Subject: [PATCH 4/6] Update PR #378 --- .github/workflows/release.yml | 24 +++++++---- npm/bt/scripts/install.js | 81 +++++++++++++++++++++-------------- npm/tests/runtime.test.cjs | 79 ++++++++++++++++++++++++++++++++-- 3 files changed, 142 insertions(+), 42 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7b324ea..85971d9d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -597,22 +597,30 @@ jobs: slack_token: ${{ secrets.SLACK_BOT_TOKEN }} slack_channel: C0ABHT0SWA2 - - name: Create GitHub release - uses: braintrustdata/sdk-actions/actions/release/create-package-github-releases@337c5475631d724441403f5a4d8ad0b32ce12e29 - with: - manifest: release-artifacts/release-manifest.json - - - name: Attach native artifacts and all npm SBOMs + - name: Publish GitHub release with all assets env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ needs.plan.outputs.tag }} run: | - # The shared action skips existing releases, including packages sharing - # one tag. Upload every SBOM here so reruns complete partial releases. + set -euo pipefail + if ! is_draft="$(gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json isDraft --jq '.isDraft')"; then + title="$(jq -r '.packages[0].release_title' release-artifacts/release-manifest.json)" + jq -r '.packages[0].release_body' release-artifacts/release-manifest.json > "$RUNNER_TEMP/notes.txt" + gh release create "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" \ + --verify-tag --draft --title "$title" --notes-file "$RUNNER_TEMP/notes.txt" + is_draft=true + fi + + # Reuse drafts on retries and repair assets on already-public releases. rm -f native-artifacts/*-dist-manifest.json gh release upload "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --clobber \ native-artifacts/* release-artifacts/*.sbom.json + # Installers and archives must be available before this becomes latest. + if [ "$is_draft" = true ]; then + gh release edit "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --draft=false + fi + smoke-install-unix: needs: - plan diff --git a/npm/bt/scripts/install.js b/npm/bt/scripts/install.js index cd9acda3..24c4fcb5 100644 --- a/npm/bt/scripts/install.js +++ b/npm/bt/scripts/install.js @@ -64,40 +64,59 @@ if (!version) { process.exit(1); } -function fetchBuffer(url, redirectsRemaining = 5) { +function fetchBuffer(url) { + let request; + let timer; return new Promise((resolve, reject) => { - https - .get(url, (response) => { - const { statusCode = 0, headers } = response; - if (statusCode >= 200 && statusCode < 300) { - const chunks = []; - response.on("data", (chunk) => chunks.push(chunk)); - response.on("end", () => resolve(Buffer.concat(chunks))); - response.on("error", reject); - return; - } - if ( - statusCode >= 300 && - statusCode < 400 && - headers.location && - redirectsRemaining > 0 - ) { + // One deadline covers connecting, receiving the body, and all redirects. + timer = setTimeout( + () => { + const error = new Error( + "download timed out after 10 minutes; retry installation", + ); + request.destroy(error); + reject(error); + }, + 10 * 60 * 1000, + ); + + function download(url, redirectsRemaining) { + request = https + .get(url, (response) => { + const { statusCode = 0, headers } = response; + if (statusCode >= 200 && statusCode < 300) { + const chunks = []; + response.on("data", (chunk) => chunks.push(chunk)); + response.on("end", () => resolve(Buffer.concat(chunks))); + response.on("error", reject); + return; + } + if ( + statusCode >= 300 && + statusCode < 400 && + headers.location && + redirectsRemaining > 0 + ) { + response.resume(); + try { + download(headers.location, redirectsRemaining - 1); + } catch (error) { + reject(error); + } + return; + } response.resume(); - fetchBuffer(headers.location, redirectsRemaining - 1).then( - resolve, - reject, + reject( + new Error( + `npm registry responded with status code ${statusCode} when downloading ${url}`, + ), ); - return; - } - response.resume(); - reject( - new Error( - `npm registry responded with status code ${statusCode} when downloading ${url}`, - ), - ); - }) - .on("error", reject); - }); + }) + .on("error", reject); + } + + download(url, 5); + }).finally(() => clearTimeout(timer)); } // Extracts a single file from an uncompressed tar archive. Tar archives are diff --git a/npm/tests/runtime.test.cjs b/npm/tests/runtime.test.cjs index 65f246a6..1bc1f989 100644 --- a/npm/tests/runtime.test.cjs +++ b/npm/tests/runtime.test.cjs @@ -172,13 +172,21 @@ async function installerWorker({ badChecksum = false, missingChecksum = false, writeError = false, + stall = null, + redirect = false, + redirectLocation = "https://registry.npmjs.org/test-redirect.tgz", } = {}) { + const { mock } = require("node:test"); + mock.timers.enable({ apis: ["setTimeout"] }); const writes = []; const messages = []; const exits = []; const requests = []; const renames = []; const removals = []; + const timeouts = []; + const destroyedRequests = []; + let clearedTimers = 0; const binary = Buffer.from("synthetic bt binary"); const tar = Buffer.alloc(1536); tar.write("package/bin/bt"); @@ -195,6 +203,9 @@ async function installerWorker({ renames, removals, binary, + timeouts, + destroyedRequests, + clearedTimers, }); const fakeRequire = (name) => { if (name === "./bt-helper") @@ -237,16 +248,27 @@ async function installerWorker({ if (name === "node:https") return { get: (url, callback) => { + new URL(url); requests.push(url); const request = new EventEmitter(); + request.destroy = (error) => { + destroyedRequests.push(url); + queueMicrotask(() => request.emit("error", error)); + }; queueMicrotask(() => { if (networkError) return request.emit("error", new Error("offline")); + const shouldRedirect = redirect && requests.length === 1; + if (!shouldRedirect && stall === "connection") return; const response = new EventEmitter(); - response.statusCode = status; - response.headers = {}; + response.statusCode = shouldRedirect ? 302 : status; + response.headers = shouldRedirect + ? { location: redirectLocation } + : {}; response.resume = () => {}; + if (shouldRedirect) mock.timers.tick(300000); callback(response); + if (shouldRedirect) return; response.emit( "data", gzipSync( @@ -257,7 +279,7 @@ async function installerWorker({ : tar, ), ); - response.emit("end"); + if (stall !== "body") response.emit("end"); }); return request; }, @@ -267,6 +289,14 @@ async function installerWorker({ vm.runInNewContext(workerData.source, { require: fakeRequire, Buffer, + setTimeout: (callback, delay) => { + timeouts.push(delay); + return setTimeout(callback, delay); + }, + clearTimeout: (timer) => { + clearedTimers++; + clearTimeout(timer); + }, console: { log: (message) => messages.push(message), error: (message) => messages.push(message), @@ -284,6 +314,16 @@ async function installerWorker({ }, }); await new Promise((resolve) => setImmediate(resolve)); + if (stall) { + mock.timers.tick((redirect ? 300000 : 600000) - 1); + await new Promise((resolve) => setImmediate(resolve)); + require("node:assert/strict").equal(destroyedRequests.length, 0); + mock.timers.tick(1); + } else { + // Completed downloads must not leave a deadline that fires later. + mock.timers.tick(600000); + } + await new Promise((resolve) => setImmediate(resolve)); finish(); } @@ -346,6 +386,37 @@ test("postinstall downloads the exact pinned binary and makes it executable", as { from: "/fallback/bt.123.tmp", to: "/fallback/bt" }, ]); assert.deepEqual(result.removals, ["/fallback/bt.123.tmp"]); + assert.deepEqual(result.timeouts, [600000]); + assert.equal(result.clearedTimers, 1); + assert.deepEqual(result.destroyedRequests, []); +}); + +test("postinstall enforces one ten-minute deadline across connections, bodies, and redirects", async () => { + for (const stall of ["connection", "body"]) { + for (const redirect of [false, true]) { + const result = await runInstaller({ stall, redirect }); + assert.equal(result.exitCode, 1); + assert.deepEqual(result.timeouts, [600000]); + assert.equal(result.clearedTimers, 1); + assert.equal(result.requests.length, redirect ? 2 : 1); + assert.deepEqual(result.destroyedRequests, [result.requests.at(-1)]); + assert.deepEqual(result.writes, []); + assert.match( + result.messages, + /download timed out after 10 minutes; retry installation/, + ); + } + } +}); + +test("postinstall clears the shared deadline after a successful redirect", async () => { + const result = await runInstaller({ redirect: true }); + assert.equal(result.exitCode, 0); + assert.equal(result.requests.length, 2); + assert.deepEqual(result.timeouts, [600000]); + assert.equal(result.clearedTimers, 1); + assert.deepEqual(result.destroyedRequests, []); + assert.equal(result.writes.length, 1); }); test("postinstall downloads the pinned version when an older hoisted binary exists", async () => { @@ -377,6 +448,7 @@ test("postinstall fails for unsupported platforms and download failures", async [{ missingVersion: true }, /cannot determine which version/], [{ status: 503 }, /status code 503/], [{ networkError: true }, /offline/], + [{ redirect: true, redirectLocation: "invalid" }, /Invalid URL/], [{ invalidTar: true }, /could not find/], [{ malformedTar: true }, /Invalid or truncated tar entry/], [{ truncatedTar: true }, /Invalid or truncated tar entry/], @@ -389,6 +461,7 @@ test("postinstall fails for unsupported platforms and download failures", async assert.equal(result.exitCode, 1); assert.equal(result.writes.length, 0); assert.equal(result.renames.length, 0); + assert.equal(result.clearedTimers, result.timeouts.length); if (options.writeError) assert.deepEqual(result.removals, ["/fallback/bt.123.tmp"]); assert.match(result.messages, message); From 7a4aa3008262890bf53aa2767971ffcef1ffb409 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:43:26 +0000 Subject: [PATCH 5/6] Update PR #378 --- npm/scripts/build-platform-packages.mjs | 38 ++++++++- npm/tests/packages.test.mjs | 106 ++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) diff --git a/npm/scripts/build-platform-packages.mjs b/npm/scripts/build-platform-packages.mjs index 9d1f92f3..35746629 100644 --- a/npm/scripts/build-platform-packages.mjs +++ b/npm/scripts/build-platform-packages.mjs @@ -95,8 +95,44 @@ for (const [target, platform] of Object.entries(targets)) { throw new Error(`Binary ${binaryName} not found at ${binPath}`); } + // Published versions are immutable and the publisher skips them on retries. + // Hash their original binary, not a rebuild (signing timestamps can differ). + const publishedUrl = `https://registry.npmjs.org/${platformPkg.name}/-/bt-${platform}-${version}.tgz`; + const published = await fetch(publishedUrl, { + signal: AbortSignal.timeout(10 * 60 * 1000), + }); + let checksumBinaryPath = binPath; + if (published.status === 404) { + await published.body?.cancel(); + } else { + if (!published.ok) { + throw new Error( + `Failed to check published package ${platformPkg.name}@${version}: HTTP ${published.status}`, + ); + } + const publishedArchive = join(stagingDir, "published.tgz"); + writeFileSync(publishedArchive, Buffer.from(await published.arrayBuffer())); + const publishedDir = join(stagingDir, "published"); + mkdirSync(publishedDir); + execFileSync( + "tar", + [ + "-xzf", + publishedArchive, + "-C", + publishedDir, + "--strip-components=1", + `package/bin/${binaryName}`, + ], + { stdio: "inherit" }, + ); + checksumBinaryPath = join(publishedDir, "bin", binaryName); + console.log( + `Using published binary checksum for ${platformPkg.name}@${version}`, + ); + } checksums[platformPkg.name] = createHash("sha256") - .update(readFileSync(binPath)) + .update(readFileSync(checksumBinaryPath)) .digest("hex"); const pkgOut = join(outDir, `bt-${platform}`); const pkgBin = join(pkgOut, "bin"); diff --git a/npm/tests/packages.test.mjs b/npm/tests/packages.test.mjs index 87891370..01469e4b 100644 --- a/npm/tests/packages.test.mjs +++ b/npm/tests/packages.test.mjs @@ -66,7 +66,14 @@ test( execFileSync("tar", ["-czf", archive, "-C", root, `bt-${target}`]); } } + const registry = join(root, "registry.mjs"); + writeFileSync( + registry, + "globalThis.fetch = async () => new Response(null, { status: 404 });\n", + ); const args = [ + "--import", + registry, join(npmDir, "scripts/build-platform-packages.mjs"), "--version", version, @@ -343,6 +350,105 @@ test( }, ); + await t.test( + "a partial release retry hashes the published Windows binary instead of its rebuild", + () => { + const [target, spec] = Object.entries(targets).find( + ([, spec]) => spec.pkg === "win32-x64", + ); + const name = `@braintrust/bt-${spec.pkg}`; + const publishedTarball = join( + artifacts, + manifest.packages.find((pkg) => pkg.name === name).tarball_asset, + ); + const rebuilt = `${stub}# changed signing timestamp\n`; + const staging = join(root, `bt-${target}`); + writeFileSync(join(staging, spec.bin), rebuilt); + execFileSync( + "python3", + [ + "-m", + "zipfile", + "-c", + join(archives, `bt-${target}.zip`), + spec.bin, + ], + { cwd: staging }, + ); + writeFileSync( + registry, + ` + import { readFileSync } from "node:fs"; + globalThis.fetch = async (url) => { + if (url === ${JSON.stringify(`https://registry.npmjs.org/${name}/-/bt-${spec.pkg}-${version}.tgz`)}) { + return new Response(readFileSync(${JSON.stringify(publishedTarball)})); + } + return new Response(null, { status: 404 }); + }; + `, + ); + const retryOut = join(root, "retry"); + execFileSync(process.execPath, [...args, "--out-dir", retryOut]); + const checksums = JSON.parse( + readFileSync(join(retryOut, "bt/checksums.json"), "utf8"), + ); + assert.equal( + readFileSync(join(retryOut, `bt-${spec.pkg}/bin/bt.exe`), "utf8"), + rebuilt, + ); + assert.equal( + checksums[name], + createHash("sha256").update(stub).digest("hex"), + ); + assert.notEqual( + checksums[name], + createHash("sha256").update(rebuilt).digest("hex"), + ); + const wrapperTarball = manifest.packages.find( + (pkg) => pkg.name === "@braintrust/bt", + ).tarball_asset; + assert.deepEqual( + JSON.parse( + execFileSync( + "tar", + [ + "-xOf", + join(retryOut, "artifacts", wrapperTarball), + "package/checksums.json", + ], + { encoding: "utf8" }, + ), + ), + checksums, + ); + }, + ); + + await t.test( + "registry and published archive failures stop before packing the wrapper", + () => { + for (const [response, message] of [ + ["new Response(null, { status: 503 })", /HTTP 503/], + ['Promise.reject(new Error("offline"))', /offline/], + ['new Response("invalid tarball")', /Command failed: tar/], + ]) { + writeFileSync( + registry, + `globalThis.fetch = async () => ${response};\n`, + ); + const failedOut = join(root, "failed"); + const result = spawnSync( + process.execPath, + [...args, "--out-dir", failedOut], + { encoding: "utf8" }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, message); + assert.equal(existsSync(join(failedOut, "bt/package.json")), false); + } + }, + ); + await t.test( "missing archives fail before emitting a publishable wrapper", () => { From 3a9404e0c34a17ad020be6f3dc4f43ed7a673685 Mon Sep 17 00:00:00 2001 From: lforst <8118419+lforst@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:14:10 +0000 Subject: [PATCH 6/6] Update PR #378 --- npm/tests/packages.test.mjs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/npm/tests/packages.test.mjs b/npm/tests/packages.test.mjs index 01469e4b..96b8675f 100644 --- a/npm/tests/packages.test.mjs +++ b/npm/tests/packages.test.mjs @@ -366,13 +366,7 @@ test( writeFileSync(join(staging, spec.bin), rebuilt); execFileSync( "python3", - [ - "-m", - "zipfile", - "-c", - join(archives, `bt-${target}.zip`), - spec.bin, - ], + ["-m", "zipfile", "-c", join(archives, `bt-${target}.zip`), spec.bin], { cwd: staging }, ); writeFileSync(