From 2aa8aafe9d0cb9de01220f2f572595d36f384a23 Mon Sep 17 00:00:00 2001 From: Raphael Vigee Date: Wed, 5 Aug 2026 10:42:39 +0200 Subject: [PATCH 1/2] feat(bench): add JS/TS Tier B scenarios, go/js/both selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends heph-bench to measure the new plugin-js the same way it already measures plugin-go, and generalizes Tier B (dist.rs) across languages instead of duplicating the go-specific path. - crates/bench-corpus: generate_js_tree produces a synthetic, network-free pnpm workspace (zero third-party deps by design — no js_install fetch, no lockfile, fully hermetic) with the same layered-DAG shape the bash and go generators already use. Wired via js_packages/js_max_depth, mirroring go_packages/go_max_depth exactly; 0 (the default) is a no-op, so existing callers get byte-identical output. - crates/bench: dist.rs rewritten around a Lang abstraction (GO/JS consts: name, provider-option fragment, package-count accessor, incrementalize fn) instead of two near-duplicate go/js code paths. `run dist --lang go|js|both` (default go, preserving existing-caller behavior exactly). `both` reports each language as its own distinct ScenarioResult ("-go"/"-js") — RunResults already carried a Vec of scenarios for exactly this, so results are never silently summed/averaged across languages. - The `corpus` CLI subcommand was missing --js-packages/--js-max-changed entirely (bench-corpus's library supported them, main.rs never exposed them) — added, mirroring --go-packages/--go-max-depth. Found and fixed a real, pre-existing bug while mirroring the go path for js: Tier B's `heph r build ///...` invocation is the two-positional form, which parses as `label("build") && ///...` — no BUILD file in the corpus sets that label, so it has always matched zero targets and exited 0, silently measuring an empty build. This affected the existing go benchmark too, not just the new js one. Fixed to the query form (`heph r -e '///...'`), verified against a real binary (built a fixture, confirmed the old form reports "matched 0 targets" and the new one actually executes targets), with a regression test that fails if the matcher regresses back to the broken form. Artifact pipeline (a real decision, not a side effect): plugin-js-cdylib is now built and published by heph.yml's `build` job alongside plugin-go-cdylib/plugin-gha-cdylib, in the same pre-release artifact bundle. No public heph-js-plugin.json manifest is generated (unlike go/gha) — bench builds its own local manifest from the raw dylib, same as it always has for go; declaring the plugin generally installable is a separate, larger decision left unmade. devenv.nix's `e2e` script deliberately does NOT build plugin-js-cdylib — there is no shipped_js_cdylib_loads bin-e2e test yet to justify the build cost, mirroring the existing lint-but-don't-e2e-test asymmetry. perf.yml wires --lang js into the perfbench job's Tier B step, gated best-effort on the baseline release actually having a js-plugin asset (older releases won't). Known, disclosed gap: no oxlint/esbuild toolchain is provisioned on any CI runner yet, so the JS Tier B comparison reports "no results" until one is — the target-matcher fix above makes that failure mode honest (real attempt, real failure) rather than the previous silent zero-targets false success. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01M3wZfyPsG8stfRQuybLRjN --- .github/workflows/heph.yml | 36 ++- .github/workflows/perf.yml | 75 ++++- crates/bench-corpus/Cargo.toml | 8 + crates/bench-corpus/src/lib.rs | 502 ++++++++++++++++++++++++++++++- crates/bench/src/dist.rs | 530 ++++++++++++++++++++++++++++----- crates/bench/src/main.rs | 313 ++++++++++++++++--- devenv.nix | 10 + 7 files changed, 1336 insertions(+), 138 deletions(-) diff --git a/.github/workflows/heph.yml b/.github/workflows/heph.yml index 6829262d..a91a42a8 100644 --- a/.github/workflows/heph.yml +++ b/.github/workflows/heph.yml @@ -260,6 +260,7 @@ jobs: PLUGIN_GO_NAME="heph-go-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" PLUGIN_GHA_NAME="heph-gha-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" PLUGIN_OCI_NAME="heph-oci-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" + PLUGIN_JS_NAME="heph-js-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" # heph-bench: the perf-regression harness (`perfbench` job). Publishing # it as a release asset alongside `heph` itself means that job never # builds anything to compare a PR against its baseline — both the @@ -277,6 +278,7 @@ jobs: echo "plugin_go_name=$PLUGIN_GO_NAME" >> $GITHUB_OUTPUT echo "plugin_gha_name=$PLUGIN_GHA_NAME" >> $GITHUB_OUTPUT echo "plugin_oci_name=$PLUGIN_OCI_NAME" >> $GITHUB_OUTPUT + echo "plugin_js_name=$PLUGIN_JS_NAME" >> $GITHUB_OUTPUT echo "bench_bin_name=$BENCH_BIN_NAME" >> $GITHUB_OUTPUT echo "debug_bin_name=$DEBUG_BIN_NAME" >> $GITHUB_OUTPUT OUT="$CARGO_TARGET_DIR/${{ matrix.target }}/release" @@ -299,20 +301,21 @@ jobs: # share nearly the whole workspace dep graph, and under the release # profile (thin-LTO + opt-level=3) each artifact's final codegen/LTO pass # is heavy — splitting into separate `cargo build` calls serializes those - # three passes and pays cargo startup + freshness re-resolution each time. + # passes and pays cargo startup + freshness re-resolution each time. # A single invocation lets cargo's jobserver overlap them (one artifact's # link tail filling cores while the next codegens). `--bin heph` selects - # the binary; `--lib` adds every selected package's lib target — i.e. both - # cdylibs (heph's own lib is already built as the bin's dependency, so it + # the binary; `--lib` adds every selected package's lib target — i.e. every + # cdylib (heph's own lib is already built as the bin's dependency, so it # costs nothing extra). `heph-bench` links the same `heph` lib crate, so # adding it here is marginal — the expensive part (engine + deps) is # already being compiled for `heph` itself. - TARGETS="--bin heph --bin heph-bench --lib -p heph -p plugin-go-cdylib -p plugin-gha-cdylib -p plugin-oci-cdylib -p bench" + TARGETS="--bin heph --bin heph-bench --lib -p heph -p plugin-go-cdylib -p plugin-gha-cdylib -p plugin-oci-cdylib -p plugin-js-cdylib -p bench" if [ "${{ matrix.os }}" = "darwin" ]; then cargo build --release --locked --target ${{ matrix.target }} $TARGETS lib="$OUT/libplugin_go_cdylib.dylib" gha_lib="$OUT/libplugin_gha_cdylib.dylib" oci_lib="$OUT/libplugin_oci_cdylib.dylib" + js_lib="$OUT/libplugin_js_cdylib.dylib" # The nix toolchain hard-links libiconv against its /nix/store path, # which is absent on user machines (dyld aborts at launch). Rewrite # those load commands to the OS /usr/lib copies — for every artifact. @@ -321,16 +324,19 @@ jobs: bash scripts/macos-portable.sh "$lib" bash scripts/macos-portable.sh "$gha_lib" bash scripts/macos-portable.sh "$oci_lib" + bash scripts/macos-portable.sh "$js_lib" else cargo zigbuild --release --locked --target ${{ matrix.target }} $TARGETS lib="$OUT/libplugin_go_cdylib.so" gha_lib="$OUT/libplugin_gha_cdylib.so" oci_lib="$OUT/libplugin_oci_cdylib.so" + js_lib="$OUT/libplugin_js_cdylib.so" fi cp "$OUT/heph-bench" $BENCH_BIN_NAME cp "$lib" $PLUGIN_GO_NAME cp "$gha_lib" $PLUGIN_GHA_NAME cp "$oci_lib" $PLUGIN_OCI_NAME + cp "$js_lib" $PLUGIN_JS_NAME # `heph` diverges into its two flavours from here — each stamped by # `patch-flavour.sh` with which one it is (`heph version` / self-upgrade @@ -425,6 +431,12 @@ jobs: name: ${{steps.build.outputs.plugin_oci_name}} path: ${{steps.build.outputs.plugin_oci_name}} + - name: Upload js plugin artifact + uses: actions/upload-artifact@v6 + with: + name: ${{steps.build.outputs.plugin_js_name}} + path: ${{steps.build.outputs.plugin_js_name}} + upload_artifacts: name: Pre-release needs: [gen, govet, build] @@ -449,8 +461,9 @@ jobs: uses: actions/download-artifact@v7 with: # Matches the CLI, both release flavours (`heph__` std, - # `heph_debug__` debug) and the go plugin cdylib - # (`heph-go-plugin__.{so,dylib}`); excludes the `repo` source artifact. + # `heph_debug__` debug) and every plugin cdylib + # (`heph-{go,gha,oci,js}-plugin__.{so,dylib}`); excludes + # the `repo` source artifact. pattern: "heph*" path: dist merge-multiple: true @@ -480,6 +493,17 @@ jobs: -prefix heph-gha-plugin -from-dir "$GITHUB_WORKSPACE/dist" -url-base "$BASE" \ -out "$GITHUB_WORKSPACE/dist/heph-gha-plugin.json" ) cat dist/heph-gha-plugin.json + # Deliberately NOT generating a heph-js-plugin.json here. The `build` + # job above now publishes the raw per-os/arch js plugin cdylib as a + # release asset (`heph-js-plugin__.{so,dylib}`) — needed so + # `perfbench`/`perf.yml` can download it for Tier B, same as go/gha. + # A public `plugins: - { identifier: { url: .../heph-js-plugin.json } }` + # manifest is a separate, larger decision (declaring the js plugin + # generally available to any real workspace, not just this bench + # pipeline's own locally-built manifest — see + # `crates/bench/src/dist.rs`'s `write_dist_config`, which builds its + # OWN manifest straight from the downloaded raw dylib and never reads + # this file). Left for a future PR to make explicitly. echo "manifest checksum: $(cat dist/heph-gha-plugin.json.sha256)" # The oci plugin (`docker_build` / `oci_pull` / `oci_push` / `oci_load`) # ships the same way — it is not compiled into the CLI. diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index cd831b18..5acb5e02 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -108,6 +108,10 @@ jobs: PERFBENCH_LAYERS: "10" PERFBENCH_FAN_OUT: "4" PERFBENCH_GO_PACKAGES: "60" + # Same corpus size as go's — no measured reason yet to size the two + # differently, and keeping them equal makes go-vs-js deltas easier to + # read at a glance. + PERFBENCH_JS_PACKAGES: "60" # 3 was too few to tell signal from CI-runner noise (the first live # runs showed exactly that). Interleaving (see the run steps below) # fixes systematic drift between candidate/baseline; more reps is what @@ -240,6 +244,21 @@ jobs: echo "no heph-bench asset in ${{ steps.baseline_release.outputs.version }} — predates publishing it, Tier A skipped" echo "bench_ok=false" >> "$GITHUB_OUTPUT" fi + # js plugin cdylib, fetched separately and best-effort for the same + # bootstrap reason as heph-bench above: baseline releases published + # before the bench js/go/both selector landed have no + # heph-js-plugin__ asset at all, and that must not fail + # this whole step (go's Tier B comparison must still run). + if gh release download "${{ steps.baseline_release.outputs.version }}" \ + --repo hephbuild/heph-artifacts-v1 \ + --dir baseline-dist-raw \ + --pattern "heph-js-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext"; then + cp "baseline-dist-raw/heph-js-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" "baseline-dist/heph-js-plugin.$ext" + echo "js_ok=true" >> "$GITHUB_OUTPUT" + else + echo "no heph-js-plugin asset in ${{ steps.baseline_release.outputs.version }} — predates publishing it, JS Tier B skipped" + echo "js_ok=false" >> "$GITHUB_OUTPUT" + fi - name: Download candidate (N) artifacts uses: actions/download-artifact@v7 @@ -255,6 +274,11 @@ jobs: cp "candidate-dist-raw/heph_${{ matrix.os }}_${{ matrix.arch }}" candidate-dist/heph cp "candidate-dist-raw/heph-bench_${{ matrix.os }}_${{ matrix.arch }}" candidate-dist/heph-bench cp "candidate-dist-raw/heph-go-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" "candidate-dist/heph-go-plugin.$ext" + # Candidate is always this same run's `build` job — no bootstrap gap + # like baseline's (see "Fetch baseline" above), so this is + # unconditional: the js plugin cdylib is always present here once + # the `build` job publishes it. + cp "candidate-dist-raw/heph-js-plugin_${{ matrix.os }}_${{ matrix.arch }}.$ext" "candidate-dist/heph-js-plugin.$ext" chmod +x candidate-dist/heph candidate-dist/heph-bench - name: Generate corpus @@ -264,6 +288,7 @@ jobs: --targets "$PERFBENCH_TARGETS" --packages "$PERFBENCH_PACKAGES" \ --layers "$PERFBENCH_LAYERS" --fan-out "$PERFBENCH_FAN_OUT" \ --go-packages "$PERFBENCH_GO_PACKAGES" \ + --js-packages "$PERFBENCH_JS_PACKAGES" \ --out bench-corpus # `heph-bench run inprocess`/`run dist` are themselves the orchestrator @@ -301,6 +326,41 @@ jobs: --out-candidate "candidate_dist_${scenario}.json" --out-baseline "baseline_dist_${scenario}.json" done + # Separate `--lang js` invocation (rather than folding into the go step + # above via `--lang both`) so `distjs` is its own tier below, gated on + # its own `js_ok` (baseline releases predating the js plugin cdylib + # must not sink the go comparison). The "Compare and report" jq merge + # below flattens `.scenarios[]` (not a hardcoded `[0]`) specifically so + # a future `--lang both` invocation — two `ScenarioResult`s per file — + # merges both instead of silently dropping the second. + # + # KNOWN GAP, disclosed rather than silently left: every generated js + # package always lists a `js_lint` target (`oxlint` by default) and, + # once it has a usable entry point, a `js_bundle` target (`esbuild`) — + # see `crates/plugin-js/src/pluginjs/provider.rs`'s `list()`. Neither + # tool is installed on these runners (no `setup-node`/`oxlint`/`esbuild` + # anywhere in this repo's workflows or `devenv.nix` as of this change), + # and this plugin has "no hermetic toolchain yet" for either (its own + # error message says so) — so this step will fail with "no oxlint/ + # esbuild binary found" every run until a js toolchain is provisioned + # for CI. `continue-on-error: true` + "Compare and report"'s missing- + # file handling absorb that the same way a missing baseline artifact + # is absorbed elsewhere in this job: the `distjs` tier reports "no + # results", nothing else in this job is affected. Provisioning that + # toolchain is a separate, larger decision (which linter/bundler + # version, hermetic vs host) left for a follow-up. + - name: Time Tier B (dist) JS scenarios + if: steps.baseline_fetch.outputs.ok == 'true' && steps.baseline_fetch.outputs.js_ok == 'true' + continue-on-error: true + run: | + for scenario in cold full-hit incremental; do + ./candidate-dist/heph-bench run dist \ + --candidate-dist candidate-dist --baseline-dist baseline-dist \ + --corpus bench-corpus --scenario "$scenario" --warmup 1 --reps "$PERFBENCH_REPS" \ + --lang js \ + --out-candidate "candidate_distjs_${scenario}.json" --out-baseline "baseline_distjs_${scenario}.json" + done + # Every scenario/tier combination is always attempted and reported — # `--allow-regression` keeps an individual `compare` call from ever # exiting non-zero, so one regression doesn't cut the table short. The @@ -330,13 +390,17 @@ jobs: echo "" echo "_FAILED: no published release found for baseline \`${{ steps.base.outputs.sha }}\`_" else - for tier in inproc dist; do + for tier in inproc dist distjs; do echo "" echo "## Tier: $tier" if [ "$tier" = "inproc" ] && [ "${{ steps.baseline_fetch.outputs.bench_ok }}" != "true" ]; then echo "_skipped: baseline release predates publishing \`heph-bench\`_" continue fi + if [ "$tier" = "distjs" ] && [ "${{ steps.baseline_fetch.outputs.js_ok }}" != "true" ]; then + echo "_skipped: baseline release predates publishing the js plugin cdylib_" + continue + fi base_files=() cand_files=() for scenario in cold full-hit incremental; do @@ -351,8 +415,13 @@ jobs: echo "_no results (run step failed — see logs)_" continue fi - jq -s '{tier: .[0].tier, scenarios: [.[].scenarios[0]]}' "${base_files[@]}" > "merged_baseline_${tier}.json" - jq -s '{tier: .[0].tier, scenarios: [.[].scenarios[0]]}' "${cand_files[@]}" > "merged_candidate_${tier}.json" + # `.scenarios[]`, not a hardcoded `.scenarios[0]`: a + # single-scenario result file (every tier today) still + # yields exactly one element, but a future `--lang both` + # result file (two `ScenarioResult`s per file) merges both + # instead of silently keeping only the first. + jq -s '{tier: .[0].tier, scenarios: [.[].scenarios[]]}' "${base_files[@]}" > "merged_baseline_${tier}.json" + jq -s '{tier: .[0].tier, scenarios: [.[].scenarios[]]}' "${cand_files[@]}" > "merged_candidate_${tier}.json" "$BENCH" compare --baseline "merged_baseline_${tier}.json" --candidate "merged_candidate_${tier}.json" \ --thresholds "$GITHUB_WORKSPACE/.github/perfbench-thresholds.json" \ --json "verdicts/${tier}.json" \ diff --git a/crates/bench-corpus/Cargo.toml b/crates/bench-corpus/Cargo.toml index 9c316837..c7f0b620 100644 --- a/crates/bench-corpus/Cargo.toml +++ b/crates/bench-corpus/Cargo.toml @@ -14,3 +14,11 @@ serde_json = "1.0" [dev-dependencies] tempfile = "3" +# Test-only: parse the generated `pnpm-workspace.yaml` / match its globs +# against generated package dirs using the exact crate+version +# `crates/plugin-js/src/pluginjs/workspace.rs` uses, so the discoverability +# assertion in `js_only_corpus_matches_manifest_and_workspace_shape` proves +# something about the real provider's matching semantics, not a hand-rolled +# approximation of them. +serde_yaml = "0.9" +wax = "0.7" diff --git a/crates/bench-corpus/src/lib.rs b/crates/bench-corpus/src/lib.rs index 87f4d217..ca83e74f 100644 --- a/crates/bench-corpus/src/lib.rs +++ b/crates/bench-corpus/src/lib.rs @@ -1,11 +1,13 @@ //! Deterministic synthetic-corpus generator for `heph-bench`. //! //! Produces a workspace tree of plain `bash`-driver targets arranged as a -//! layered DAG (bounded fan-out, no cycles), plus an optional `go/` subtree -//! of hermetic Go packages for scenarios that must cross the real -//! plugin-cdylib seam. The go packages are discovered from `go.mod`; the only -//! BUILD file written there declares the build variant, without which the go -//! provider lists no build targets at all (see `write_go_variant_build`). +//! layered DAG (bounded fan-out, no cycles), plus optional `go/` and `js/` +//! subtrees for scenarios that must cross the real plugin-cdylib seam. The +//! go packages are discovered from `go.mod`; the only BUILD file written +//! there declares the build variant, without which the go provider lists no +//! build targets at all (see `write_go_variant_build`). The js packages are +//! a hermetic pnpm workspace, auto-discovered by the js provider with no +//! BUILD file needed at all. //! //! Same seed + same params => byte-identical tree. A CI run generates the //! corpus once (from the PR-head generator) and points both the baseline and @@ -67,6 +69,19 @@ pub struct CorpusParams { /// containing its `go.mod`). Defaults to the copy in this workspace via /// [`default_gorepogen_dir`]. pub gorepogen_dir: PathBuf, + /// Number of JS/TS packages to generate under `js/` (Tier B, plugin-js). + /// 0 = no js subtree. Unlike `go_packages`, generation never shells out + /// to an external tool — see [`generate_js_tree`]'s doc comment for why + /// there is nothing external to resolve. + pub js_packages: usize, + /// Max nesting depth, in directories below `js/packages/`, a generated + /// package can sit at — mirrors `go_max_depth`'s role of bounding tree + /// shape, translated into `pnpm-workspace.yaml` glob patterns (one + /// literal `*` path segment per depth level: `packages/*`, + /// `packages/*/*`, ...) since pnpm globs don't cross directory + /// boundaries (see `crates/plugin-js/src/pluginjs/workspace.rs`'s + /// `resolve_members` doc comment). + pub js_max_depth: usize, } impl Default for CorpusParams { @@ -80,6 +95,8 @@ impl Default for CorpusParams { go_packages: 0, go_max_depth: 4, gorepogen_dir: default_gorepogen_dir(), + js_packages: 0, + js_max_depth: 4, } } } @@ -116,6 +133,12 @@ pub struct CorpusManifest { /// The BUILD package directories that hold bash targets — the mutation /// unit for [`incrementalize`]. pub bash_packages: Vec, + /// Number of js packages generated under `js/` (0 if `js_packages == 0`). + pub js_package_count: usize, + /// Package-path prefix covering the generated js subtree (`"js"`), for a + /// `//js/...`-shaped build against a real pnpm workspace, dlopening the + /// real js plugin cdylib. + pub js_prefix: String, } struct Layered { @@ -240,12 +263,21 @@ pub fn generate(params: &CorpusParams, root: &Path) -> Result { 0 }; + let js_package_count = if params.js_packages > 0 { + generate_js_tree(params, root)?; + params.js_packages + } else { + 0 + }; + let manifest = CorpusManifest { bash_addrs: g.addrs, bash_prefix: String::new(), go_package_count, go_prefix: "go".to_string(), bash_packages, + js_package_count, + js_prefix: "js".to_string(), }; save_manifest(&manifest, root)?; Ok(manifest) @@ -358,6 +390,174 @@ fn write_go_variant_build(go_root: &Path) -> Result<()> { Ok(()) } +/// A `js/` subtree the js provider auto-discovers via `pnpm-workspace.yaml` + +/// each package's own `package.json` (no BUILD file) — generated directly in +/// Rust rather than shelled out to an external tool, unlike +/// [`generate_go_tree`]. There is nothing to resolve at generation (or +/// measurement) time: the generated tree declares zero third-party npm +/// dependencies — no `dependencies`/`devDependencies` entry names a real npm +/// package at all — so every Tier B JS scenario stays fully offline (no +/// `js_install` network fetch, no lockfile needed), the same "resolve once, +/// up front" principle `generate_go_tree`'s `go mod tidy` establishes, just +/// simpler here since there is nothing external at all. Cross-package edges +/// are relative TypeScript `import` statements instead, so the module DAG is +/// real without a package manager ever needing to resolve anything. +/// +/// Package layout mirrors the bash-target DAG's own shape +/// ([`layer_targets`]/[`generate`]): `params.layers` layers, each package +/// depending on up to `params.fan_out` packages in the layer below (picked +/// with the same [`Rng`] machinery), so js and bash corpora are structurally +/// comparable at the same params. The RNG here is seeded independently of +/// the bash-target loop's — `params.seed` XORed with a distinguishing +/// constant, same convention [`incrementalize_go`] uses — so enabling or +/// disabling `js_packages` never perturbs the bash (or go) output, and vice +/// versa. +/// +/// Chose `pnpm-workspace.yaml` over npm's `package.json` `"workspaces"` +/// array (`crates/plugin-js/src/pluginjs/workspace.rs` supports both): pnpm +/// needs no root `package.json` at all — the workspace-member glob list +/// lives in its own file — one fewer file to generate, while still +/// exercising the identical glob-based discovery path the npm branch would. +fn generate_js_tree(params: &CorpusParams, root: &Path) -> Result<()> { + let js_root = root.join("js"); + std::fs::create_dir_all(&js_root).context("create js/")?; + + let layers = params.layers.max(1); + let max_depth = params.js_max_depth.max(1); + let n = params.js_packages; + + // Independent of the bash-target loop's `rng` above — see doc comment. + let mut rng = Rng::new(params.seed ^ 0xBA5E_1000_ABCD_EF01); + + // Layer and nesting depth are deterministic functions of the package + // index (not rng-drawn), so the tree's shape never shifts under however + // many `rng` draws the dep-picking loop below ends up making. + let mut by_layer: Vec> = vec![Vec::new(); layers]; + let mut dirs: Vec> = Vec::with_capacity(n); + for i in 0..n { + by_layer + .get_mut(i % layers) + .expect("i % layers is always < by_layer.len() == layers") + .push(i); + + let depth = i % max_depth; + let mut comps = vec!["packages".to_string()]; + for d in 0..depth { + comps.push(format!("grp{d}")); + } + comps.push(format!("pkg{i}")); + dirs.push(comps); + } + + for i in 0..n { + let layer = i % layers; + let deps: Vec = if layer == 0 { + Vec::new() + } else { + let below = by_layer + .get(layer - 1) + .expect("layer > 0 here, so layer - 1 < layers == by_layer.len()"); + let k = params.fan_out.min(below.len()); + let mut picked = Vec::with_capacity(k); + for _ in 0..k { + let idx = *below + .get(rng.below(below.len())) + .expect("rng.below(below.len()) < below.len()"); + if !picked.contains(&idx) { + picked.push(idx); + } + } + picked + }; + + let dir_comps = dirs + .get(i) + .expect("i < n == dirs.len() by the loop bound above"); + let pkg_dir = js_root.join(dir_comps.join("/")); + std::fs::create_dir_all(&pkg_dir) + .with_context(|| format!("create {}", pkg_dir.display()))?; + + let package_json = serde_json::json!({ + "name": format!("corpus-js-pkg{i}"), + "version": "0.0.0", + "main": "./index.ts", + }); + std::fs::write( + pkg_dir.join("package.json"), + serde_json::to_vec_pretty(&package_json).context("encode package.json")?, + ) + .with_context(|| format!("write {}/package.json", pkg_dir.display()))?; + + let mut src = String::new(); + for (n_dep, &dep) in deps.iter().enumerate() { + let dep_comps = dirs + .get(dep) + .expect("dep is a target index from by_layer, always < dirs.len()"); + let import_path = relative_ts_import(dir_comps, dep_comps); + writeln!( + src, + "import {{ value as dep{n_dep} }} from \"{import_path}\";" + ) + .context("format import")?; + } + // Same principle as the bash targets' `run` body: the module never + // reads a dep's real content, only imports its exported binding — + // dependency cost comes entirely from the import edges wiring the + // module graph (parse, resolve, first-party closure walk), not from + // any payload. + let value_expr = if deps.is_empty() { + format!("{i}") + } else { + let terms = (0..deps.len()) + .map(|n_dep| format!("dep{n_dep}")) + .collect::>() + .join(" + "); + format!("{i} + {terms}") + }; + writeln!(src, "export const value: number = {value_expr};").context("format export")?; + std::fs::write(pkg_dir.join("index.ts"), src) + .with_context(|| format!("write {}/index.ts", pkg_dir.display()))?; + } + + // One glob pattern per depth level actually reachable (0..max_depth): a + // literal `*` path segment per nesting level, matching pnpm's own + // non-crossing glob semantics (`packages/*` never matches + // `packages/grp0/pkg5` — see `workspace.rs`'s `resolve_members` doc + // comment) — every generated package is covered by exactly one line. + let mut yaml = String::from("packages:\n"); + for d in 0..max_depth { + let mut pattern = String::from("packages"); + for _ in 0..=d { + pattern.push_str("/*"); + } + writeln!(yaml, " - \"{pattern}\"").context("format pnpm-workspace.yaml pattern")?; + } + std::fs::write(js_root.join("pnpm-workspace.yaml"), yaml) + .context("write js/pnpm-workspace.yaml")?; + + Ok(()) +} + +/// Relative TS import specifier from the package at `from_dir` to the +/// package at `to_dir` (both directory-component lists relative to `js/`), +/// pointing at the target's `index.ts` module (extension omitted — standard +/// relative-import style). Always anchored (`./` or `../`), never a bare +/// specifier, so it is never mistaken for a package-name import. +fn relative_ts_import(from_dir: &[String], to_dir: &[String]) -> String { + let common = from_dir + .iter() + .zip(to_dir.iter()) + .take_while(|(a, b)| a == b) + .count(); + let mut parts: Vec = (common..from_dir.len()).map(|_| "..".to_string()).collect(); + parts.extend(to_dir.iter().skip(common).cloned()); + if parts.first().map(String::as_str) != Some("..") { + parts.insert(0, ".".to_string()); + } + parts.push("index".to_string()); + parts.join("/") +} + /// `ceil(len * fraction)`, clamped to `len`. The f64->usize cast is provably /// non-negative (product of two non-negative factors, then `ceil`), but /// clippy's `cast_sign_loss` can't see that from the call site. @@ -448,6 +648,53 @@ pub fn incrementalize_go(go_root: &Path, fraction: f64, seed: u64) -> Result) -> Result<()> { + if !dir.exists() { + return Ok(()); + } + for entry in std::fs::read_dir(dir).with_context(|| format!("read dir {}", dir.display()))? { + let entry = entry.context("read dir entry")?; + let path = entry.path(); + if entry.file_type().context("get file type")?.is_dir() { + collect_ts_files(&path, out)?; + } else if path.extension().and_then(|e| e.to_str()) == Some("ts") { + out.push(path); + } + } + Ok(()) +} + +/// Same idea as [`incrementalize_go`], for the js subtree: appends a +/// trailing top-level statement (valid anywhere in a `.ts` module, changes +/// nothing about the exported API) to a deterministic fraction of `.ts` +/// files under `js_root`. Unlike Go there is no test-file convention to +/// exclude — [`generate_js_tree`] never writes one — so, unlike +/// [`incrementalize_go`], every `.ts` file found is eligible. +pub fn incrementalize_js(js_root: &Path, fraction: f64, seed: u64) -> Result { + let mut files = Vec::new(); + collect_ts_files(js_root, &mut files)?; + files.sort(); + + let mut rng = Rng::new(seed ^ 0xF00D_F00D_ABCD_1234); + let n = fraction_count(files.len(), fraction); + let mut touched = std::collections::HashSet::new(); + while touched.len() < n { + touched.insert(rng.below(files.len())); + } + + for &idx in &touched { + let path = files + .get(idx) + .expect("idx drawn from files.len() in the loop above"); + let mut src = + std::fs::read_to_string(path).with_context(|| format!("read {}", path.display()))?; + src.push_str(&format!("\n// bench-mutated seed={seed}\n")); + std::fs::write(path, src).with_context(|| format!("rewrite {}", path.display()))?; + } + + Ok(touched.len()) +} + #[cfg(test)] mod tests { use super::*; @@ -649,4 +896,249 @@ mod tests { .expect("read test file"); assert_eq!(test_src, "package pkg\n", "test file must not be mutated"); } + + /// Recursively collect `(path relative to `root`, contents)` under + /// `root`, sorted — used by the js-tree tests below the same way + /// `generate_is_deterministic`'s local `read_all` is used for the bash + /// tree, just recursive (js packages nest per `js_max_depth`). + fn read_all_recursive(root: &Path) -> Vec<(String, String)> { + fn walk(dir: &Path, root: &Path, out: &mut Vec<(String, String)>) { + let entries = std::fs::read_dir(dir).expect("read_dir"); + for entry in entries { + let entry = entry.expect("dir entry"); + let path = entry.path(); + if path.is_dir() { + walk(&path, root, out); + } else { + let rel = path.strip_prefix(root).unwrap_or(&path); + out.push(( + rel.to_string_lossy().into_owned(), + std::fs::read_to_string(&path).expect("read file"), + )); + } + } + } + let mut out = Vec::new(); + walk(root, root, &mut out); + out.sort(); + out + } + + /// Recursively collect directories under `root` that contain their own + /// `package.json`, as paths relative to `root` — a minimal stand-in for + /// `collect_js_packages` (kept local to this test rather than depending + /// on the `plugin-js` crate, per this crate's existing "no plugin-crate + /// dependency" shape — see `generate_go_tree`'s doc comment). + fn discover_package_dirs(dir: &Path, root: &Path, out: &mut Vec) { + if dir.join("package.json").is_file() { + out.push(dir.strip_prefix(root).unwrap_or(dir).to_path_buf()); + } + let entries = std::fs::read_dir(dir).expect("read_dir"); + for entry in entries { + let path = entry.expect("dir entry").path(); + if path.is_dir() { + discover_package_dirs(&path, root, out); + } + } + } + + #[test] + fn js_only_corpus_matches_manifest_and_workspace_shape() { + let params = CorpusParams { + seed: 5, + target_count: 20, + packages: 4, + layers: 3, + fan_out: 2, + js_packages: 12, + js_max_depth: 2, + ..Default::default() + }; + let dir = tempfile::tempdir().expect("tempdir"); + let manifest = generate(¶ms, dir.path()).expect("generate"); + + assert_eq!(manifest.js_package_count, 12); + assert_eq!(manifest.js_prefix, "js"); + // go stays untouched: js_packages > 0 alone must not synthesize a go + // subtree or count. + assert_eq!(manifest.go_package_count, 0); + assert!(!dir.path().join("go").exists()); + + let js_root = dir.path().join("js"); + + // Shape assertion: parse the generated `pnpm-workspace.yaml` with the + // exact struct shape `workspace.rs::read_pnpm_workspace_globs` reads + // (a `packages: Vec` list, see its test fixtures), then match + // every discovered package dir against those globs with the same + // `wax` crate/version the real provider matches with. + #[derive(serde::Deserialize)] + struct PnpmWorkspaceFile { + #[serde(default)] + packages: Vec, + } + let raw = std::fs::read_to_string(js_root.join("pnpm-workspace.yaml")) + .expect("read pnpm-workspace.yaml"); + let parsed: PnpmWorkspaceFile = + serde_yaml::from_str(&raw).expect("parse pnpm-workspace.yaml"); + assert!(!parsed.packages.is_empty()); + + use wax::Program as _; + let globs: Vec = parsed + .packages + .iter() + .map(|p| wax::Glob::new(p).expect("valid glob")) + .collect(); + + let mut pkg_dirs = Vec::new(); + discover_package_dirs(&js_root, &js_root, &mut pkg_dirs); + assert_eq!( + pkg_dirs.len(), + 12, + "every generated package must be found on disk" + ); + + // js_max_depth == 2 must actually produce nesting beyond the flat + // `packages/pkgN` layer — otherwise this test would pass trivially + // with just one glob line. + assert!( + pkg_dirs.iter().any(|d| d.components().count() > 2), + "expected at least one package nested below packages/" + ); + + for rel in &pkg_dirs { + assert!( + globs.iter().any(|g| g.is_match(rel.as_path())), + "{} matched by no pnpm-workspace.yaml glob", + rel.display() + ); + + let pj_raw = std::fs::read_to_string(js_root.join(rel).join("package.json")) + .expect("read package.json"); + let pj: serde_json::Value = serde_json::from_str(&pj_raw).expect("parse package.json"); + assert!( + pj.get("name").and_then(serde_json::Value::as_str).is_some(), + "{}: package.json missing name", + rel.display() + ); + assert!( + pj.get("main").and_then(serde_json::Value::as_str).is_some(), + "{}: package.json missing main", + rel.display() + ); + // Hermeticity: no entry may name a real npm package. + assert!(pj.get("dependencies").is_none()); + assert!(pj.get("devDependencies").is_none()); + } + } + + #[test] + fn bash_and_go_output_unaffected_by_js_packages() { + let base = CorpusParams { + seed: 9, + target_count: 30, + packages: 6, + layers: 3, + fan_out: 2, + ..Default::default() + }; + let without_js = base.clone(); + let mut with_js = base; + with_js.js_packages = 8; + with_js.js_max_depth = 3; + + let a = tempfile::tempdir().expect("tempdir"); + let b = tempfile::tempdir().expect("tempdir"); + let manifest_a = generate(&without_js, a.path()).expect("generate a"); + let manifest_b = generate(&with_js, b.path()).expect("generate b"); + + assert_eq!(manifest_a.bash_addrs, manifest_b.bash_addrs); + assert_eq!(manifest_a.bash_packages, manifest_b.bash_packages); + for pkg in &manifest_a.bash_packages { + let src_a = std::fs::read_to_string(a.path().join(pkg).join("BUILD")).expect("read a"); + let src_b = std::fs::read_to_string(b.path().join(pkg).join("BUILD")).expect("read b"); + assert_eq!( + src_a, src_b, + "bash BUILD output for {pkg} must be unaffected by js_packages" + ); + } + + assert_eq!(manifest_a.js_package_count, 0); + assert_eq!(manifest_b.js_package_count, 8); + } + + #[test] + fn js_tree_generation_is_deterministic() { + let params = CorpusParams { + seed: 77, + target_count: 10, + packages: 2, + layers: 2, + fan_out: 1, + js_packages: 15, + js_max_depth: 3, + ..Default::default() + }; + let a = tempfile::tempdir().expect("tempdir"); + let b = tempfile::tempdir().expect("tempdir"); + generate(¶ms, a.path()).expect("generate a"); + generate(¶ms, b.path()).expect("generate b"); + + let js_a = read_all_recursive(&a.path().join("js")); + let js_b = read_all_recursive(&b.path().join("js")); + assert!(!js_a.is_empty()); + assert_eq!(js_a, js_b); + } + + #[test] + fn incrementalize_js_touches_requested_fraction() { + let params = CorpusParams { + seed: 3, + target_count: 10, + packages: 2, + layers: 2, + fan_out: 1, + js_packages: 10, + js_max_depth: 2, + ..Default::default() + }; + let dir = tempfile::tempdir().expect("tempdir"); + generate(¶ms, dir.path()).expect("generate"); + let js_root = dir.path().join("js"); + + let touched = incrementalize_js(&js_root, 0.3, 55).expect("incrementalize_js"); + assert_eq!(touched, 3); // ceil(10 * 0.3), one index.ts per package + + let mutated = read_all_recursive(&js_root) + .into_iter() + .filter(|(path, contents)| { + path.ends_with(".ts") && contents.contains("bench-mutated seed=55") + }) + .count(); + assert_eq!(mutated, 3); + } + + // Ignored like `go_tree_produces_requested_package_count`: `go mod tidy` + // over gorepogen's output needs network. Extends that scenario to prove + // go and js subtrees coexist independently in one corpus. + #[test] + #[ignore = "needs network for `go mod tidy` on gorepogen's third-party imports"] + fn go_and_js_together_produce_both_independently() { + let params = CorpusParams { + seed: 3, + target_count: 30, + packages: 3, + layers: 3, + fan_out: 2, + go_packages: 5, + js_packages: 6, + js_max_depth: 2, + ..Default::default() + }; + let dir = tempfile::tempdir().expect("tempdir"); + let manifest = generate(¶ms, dir.path()).expect("generate"); + assert_eq!(manifest.go_package_count, 5); + assert_eq!(manifest.js_package_count, 6); + assert!(dir.path().join("go/go.mod").is_file()); + assert!(dir.path().join("js/pnpm-workspace.yaml").is_file()); + } } diff --git a/crates/bench/src/dist.rs b/crates/bench/src/dist.rs index beb6ad03..2b49abd6 100644 --- a/crates/bench/src/dist.rs +++ b/crates/bench/src/dist.rs @@ -1,6 +1,7 @@ //! Tier B: the real, prebuilt `heph` binary spawned as a child process, -//! dlopening the real `go` plugin cdylib — the seam `crates/bin-e2e` exists -//! to cover and an in-process test structurally cannot reach. +//! dlopening the real per-language plugin cdylib(s) — the seam +//! `crates/bin-e2e` exists to cover and an in-process test structurally +//! cannot reach. //! //! Only prebuilt artifacts are used, located the same way //! `crates/bin-e2e/tests/common/mod.rs`'s `Dist` does: a normalized @@ -12,6 +13,18 @@ //! per-commit "subject" binary to keep a stable contract with. `prepare`/ //! `measure_once` still split the same way as `inprocess`'s for a uniform //! orchestrator shape, not because a compatibility seam requires it here. +//! +//! **Languages**: [`GO`] and [`JS`] are the two [`Lang`]s this tier knows how +//! to build — one `///...` corpus subtree, one plugin cdylib, one +//! provider-level option a real workspace has no default for (mirrors the go +//! provider's required `gotool`; see `Lang`'s doc). A caller picks one or +//! both via the `langs` slice threaded through [`prepare`]/[`measure_once`]; +//! `crates/bench/src/main.rs`'s `--lang` flag is what turns that into a CLI +//! selector. Each language selects its own targets its own way (see +//! `Lang::label`'s doc): go carries a `go-build` label the provider stamps +//! on every compile target; js has no such label mechanism, so it uses the +//! `-e ''` form instead. Both go through the same `matched_targets` +//! safety net regardless of which form selected them. use anyhow::{Context, Result, bail}; use bench_corpus::CorpusManifest; @@ -36,7 +49,7 @@ impl Dist { if !heph.is_file() { bail!("{} does not contain a `heph` binary", dir.display()); } - // Absolute: `build_go_tree` spawns `heph` with `current_dir(corpus)`, + // Absolute: `build_lang_tree` spawns `heph` with `current_dir(corpus)`, // so a relative `--dist` path (the common case — CI passes plain // `candidate-dist`) would silently resolve against the corpus dir // instead of the caller's cwd once that happens. @@ -77,53 +90,129 @@ fn host_arch() -> &'static str { } } -/// Write the go-plugin manifest + a `.hephconfig` pointing at it, into -/// `corpus` — a real config a real `heph` invocation loads, forcing the -/// dlopen + ABI-negotiation + checksum-verify path. `corpus` must already be -/// absolute (see `Dist::locate`'s comment for why). -fn write_go_config(corpus: &Path, dist: &Dist) -> Result<()> { - let dylib = dist.plugin("go"); - if !dylib.is_file() { - bail!( - "missing {} — the dist dir must contain the go plugin cdylib \ - (heph-go-plugin.{DYLIB_EXT}), not just the `heph` binary", - dylib.display() - ); - } - let manifest_path = corpus.join("heph-go-plugin.json"); - let sum = sha256_file(&dylib)?; - let doc = serde_json::json!({ - "name": "go", - "version": "bench", - "artifacts": [{ - "os": host_os(), - "arch": host_arch(), - "path": dylib, - "checksum": sum, - }], - }); - std::fs::write(&manifest_path, serde_json::to_vec_pretty(&doc)?) - .with_context(|| format!("write {}", manifest_path.display()))?; - - // `gotool: host` — the go provider requires an explicit choice (host / - // pinned version / a toolchain-producing target) and has no default. +/// A language Tier B knows how to build: `name` doubles as the plugin +/// identity (`heph--plugin.`) *and* the corpus subtree / target +/// prefix (`///...`) — true for both `go/` and `js/`, the only two +/// today, and simpler than carrying two separate fields that would always +/// agree in practice. +/// +/// Deliberately data, not a `match` on a `&str` language tag scattered +/// across this module — adding a third language means adding one more +/// `const`, not hunting down every place a go-vs-js branch was hand-written. +#[derive(Debug)] +pub struct Lang { + pub name: &'static str, + /// The provider's `options:` fragment this language's plugin requires + /// with no default — mirrors the go provider's `gotool` option (see + /// `write_dist_config`'s doc comment) and the js provider's `pkgmanager` + /// option (`crates/plugin-js/src/pluginjs/provider.rs`: "required — set + /// it to \"npm\" or \"pnpm\"", no genuine default to pick for a real + /// workspace). + provider_options: &'static str, + /// The label this language's *compile* targets carry, if any — go's + /// `go-build` label (`crates/plugin-go/src/plugingo/driver_compile.rs` + /// et al.) lets target selection use the fast `heph r