From dc805a71e6125b34d490ad17546217872999abb4 Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Thu, 11 Jun 2026 22:37:11 +0000 Subject: [PATCH 1/5] fix: harden brainstorm evaluation score parsing --- crates/refinery_core/src/brainstorm.rs | 236 +++++++++++++++++- docs/HANDOFF.md | 7 +- ...rch-brainstorm-strategy-benchmarks-plan.md | 1 + todos/013-brainstorm-strategy-benchmarks.md | 6 +- 4 files changed, 235 insertions(+), 15 deletions(-) diff --git a/crates/refinery_core/src/brainstorm.rs b/crates/refinery_core/src/brainstorm.rs index dc77342..15c32e0 100644 --- a/crates/refinery_core/src/brainstorm.rs +++ b/crates/refinery_core/src/brainstorm.rs @@ -296,16 +296,7 @@ fn parse_brainstorm_evaluation_response(response: &str) -> Option(json).ok()) .or_else(|| serde_json::from_str::(response).ok())?; - #[allow(clippy::cast_precision_loss)] - let score = parsed - .get("score") - .and_then(|v| { - v.as_u64() - .map(|u| u as f64) - .or_else(|| v.as_f64()) - .or_else(|| v.as_str().and_then(|s| s.parse::().ok())) - }) - .filter(|score| (1.0..=10.0).contains(score))?; + let score = parse_brainstorm_score(&parsed).filter(|score| (1.0..=10.0).contains(score))?; let rationale = parsed .get("rationale") .and_then(|value| value.as_str()) @@ -315,6 +306,112 @@ fn parse_brainstorm_evaluation_response(response: &str) -> Option Option { + parsed + .get("score") + .and_then(score_value_as_f64) + .or_else(|| parsed.get("overall_score").and_then(score_value_as_f64)) + .or_else(|| average_dimension_scores(parsed)) +} + +fn score_value_as_f64(value: &serde_json::Value) -> Option { + #[allow(clippy::cast_precision_loss)] + value + .as_u64() + .map(|u| u as f64) + .or_else(|| value.as_f64()) + .or_else(|| value.as_str().and_then(parse_score_text)) + .or_else(|| value.get("score").and_then(score_value_as_f64)) + .or_else(|| value.get("value").and_then(score_value_as_f64)) +} + +fn parse_score_text(text: &str) -> Option { + let trimmed = text.trim(); + trimmed.parse::().ok().or_else(|| { + let number_spans = number_spans(trimmed); + if number_spans.is_empty() { + return None; + } + + let lower = trimmed.to_ascii_lowercase(); + if let Some(score_idx) = lower.rfind("score") { + if let Some((start, end)) = number_spans + .iter() + .copied() + .find(|(start, _)| *start >= score_idx) + { + return trimmed[start..end].parse::().ok(); + } + } + + if let Some((start, end)) = number_spans.first().copied() { + let after_first = lower[end..].trim_start(); + if after_first.starts_with("out of") || after_first.starts_with("/10") { + return trimmed[start..end].parse::().ok(); + } + } + + let (start, end) = number_spans.last().copied()?; + trimmed[start..end].parse::().ok() + }) +} + +fn number_spans(text: &str) -> Vec<(usize, usize)> { + let chars: Vec<(usize, char)> = text.char_indices().collect(); + let mut spans = Vec::new(); + let mut idx = 0; + + while idx < chars.len() { + let (start, ch) = chars[idx]; + let next = chars.get(idx + 1).map(|(_, next)| *next); + let begins_number = ch.is_ascii_digit() + || ((ch == '-' || ch == '+') && next.is_some_and(|next| next.is_ascii_digit())); + if !begins_number { + idx += 1; + continue; + } + + let mut cursor = idx; + if ch == '-' || ch == '+' { + cursor += 1; + } + + let mut has_dot = false; + while cursor < chars.len() { + let (_, current) = chars[cursor]; + if current.is_ascii_digit() { + cursor += 1; + } else if current == '.' && !has_dot { + has_dot = true; + cursor += 1; + } else { + break; + } + } + + let end = chars + .get(cursor) + .map_or_else(|| text.len(), |(end, _)| *end); + spans.push((start, end)); + idx = cursor; + } + + spans +} + +fn average_dimension_scores(parsed: &serde_json::Value) -> Option { + let scores = ["originality", "insight", "depth", "feasibility"] + .into_iter() + .map(|field| parsed.get(field).and_then(score_value_as_f64)) + .collect::>>()?; + + if scores.iter().all(|score| (1.0..=10.0).contains(score)) { + Some(scores.iter().sum::() / 4.0) + } else { + None + } +} + fn sanitize_brainstorm_context(text: &str) -> String { let mut sanitized = prompts::sanitize_for_score_tag(text); for tag in [ @@ -1314,6 +1411,125 @@ mod tests { assert_eq!(parsed.rationale, "good tension"); } + #[test] + fn parse_brainstorm_evaluation_accepts_score_text_with_scale() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"8 out of 10"}"#, + ) + .expect("scaled score text should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_accepts_slash_scale_without_using_denominator() { + let parsed = + parse_brainstorm_evaluation_response(r#"{"rationale":"good tension","score":"8/10"}"#) + .expect("slash-scale score text should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_accepts_plus_signed_score_text() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"score: +8"}"#, + ) + .expect("plus-signed score text should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_uses_trailing_score_after_scale_text() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"on a 1-10 scale: 8"}"#, + ) + .expect("trailing score after scale text should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_prefers_number_after_score_label() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"on a 1-10 scale, score: 8"}"#, + ) + .expect("number after score label should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_accepts_nested_score_value() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":{"value":7.5}}"#, + ) + .expect("nested score value should parse"); + + assert!((parsed.score - 7.5).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_accepts_overall_score_alias() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","overall_score":9}"#, + ) + .expect("overall_score alias should parse"); + + assert!((parsed.score - 9.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_averages_dimensions_when_overall_score_missing() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"originality":8,"insight":7,"depth":9,"feasibility":6,"rationale":"good tension"}"#, + ) + .expect("dimension scores should provide fallback score"); + + assert!((parsed.score - 7.5).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_rejects_missing_fallback_dimension() { + assert!( + parse_brainstorm_evaluation_response( + r#"{"originality":8,"insight":7,"depth":9,"rationale":"missing feasibility"}"#, + ) + .is_none() + ); + } + + #[test] + fn parse_brainstorm_evaluation_rejects_extra_score_without_required_dimension() { + assert!( + parse_brainstorm_evaluation_response( + r#"{"originality":8,"insight":7,"depth":9,"novelty":10,"rationale":"extra score cannot replace feasibility"}"#, + ) + .is_none() + ); + } + + #[test] + fn parse_brainstorm_evaluation_rejects_out_of_range_text_score() { + assert!( + parse_brainstorm_evaluation_response( + r#"{"rationale":"negative text score","score":"-1 out of 10"}"#, + ) + .is_none() + ); + } + + #[test] + fn parse_brainstorm_evaluation_rejects_out_of_range_fallback_dimensions() { + assert!( + parse_brainstorm_evaluation_response( + r#"{"originality":8,"insight":7,"depth":11,"feasibility":6,"rationale":"bad score"}"#, + ) + .is_none() + ); + } + #[test] fn prompt_variant_strategy_parsing_accepts_supported_variants() { assert_eq!( diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 3046fd7..39986ef 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -2,7 +2,7 @@ Current state of the project and active work. Read this at session start. Update before compaction or at natural breakpoints. -**Last updated:** 2026-06-09 +**Last updated:** 2026-06-11 ## Project State @@ -45,7 +45,7 @@ See `memory/verb_architecture.md` for full taxonomy with consistent terminology. Check `todos/` for the full list. Key ones: -- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, benchmark-only iteration variants, L2 six-prompt variant suite, blind review pack, first-pass qualitative L2 panel review, hidden L3 prompt-reframing implementation, 3-model L3 smoke, updated-model 2-model L3 smoke, and two-prompt 3-model L3 sample completed; next step is either 2-4 more L3 prompts with Codex/GLM/Kimi-for-coding or triage GLM invalid eval scores on expanded runs; avoid full MiniMax M3 suites until runtime/output budget controls are explicit +- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, benchmark-only iteration variants, L2 six-prompt variant suite, blind review pack, first-pass qualitative L2 panel review, hidden L3 prompt-reframing implementation, 3-model L3 smoke, updated-model 2-model L3 smoke, two-prompt 3-model L3 sample, and initial verified parser hardening for recoverable GLM-style invalid eval scores completed; next step is either live-validate the parser hardening with 2-4 more L3 prompts using Codex/GLM/Kimi-for-coding or continue deeper GLM triage if invalid eval scores persist; avoid full MiniMax M3 suites until runtime/output budget controls are explicit - **025** — optional brainstorm lineage-reference polish if softer phrases like "builds on..." feel too process-oriented in demos - **018** — brainstorm divergence expansion: first-stage prompt reframing implemented behind hidden `brainstorm --prompt-variants per-model`; next run L3 benchmarks and defer domain collisions - **021** — evaluate TOON (`toon-format/toon`) for prompt-facing artifact export / benchmark fixtures @@ -59,6 +59,7 @@ Triage pattern: fix P1/P2 with code, create TODOs for P3/nitpicks, reply to ever ## Recent Context +- 2026-06-11 GLM invalid-evaluation parser hardening completed (`todos/013`, plan `docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md`). `parse_brainstorm_evaluation_response()` now accepts recoverable score variants seen/plausible in expanded brainstorm evals: scaled score text like `"8 out of 10"`, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores. It rejects incomplete dimension sets even if extra numeric fields are present. Verified with `cargo fmt --all -- --check`, `cargo test -p refinery_core parse_brainstorm_evaluation -q`, `cargo test -p refinery_core brainstorm -q`, and `cargo clippy -p refinery_core --all-targets -- -D warnings`. Next useful validation is a live Pi-backed L3 run with Codex/GLM/Kimi-for-coding; if invalid eval scores persist, capture/preserve raw invalid responses for deeper GLM triage. - 2026-06-09 three-model L3 sample completed (`todos/013`, `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`) after PR #44 merged. Compared `--prompt-variants off` vs `per-model` on product and technical prompts using `pi/openai-codex/gpt-5.4:off`, `pi/zai/glm-5.1:off`, and `pi/kimi-coding/kimi-for-coding:off`, serial with `--max-concurrent 1`. Baseline runs completed clean (`18` calls each, ~7-8m). Per-model runs completed with full 12-candidate final sets but degraded evaluation status (`75` calls each, ~30-38m): product had one GLM invalid eval score; technical had one Codex SSE response-header timeout and one GLM invalid eval score. `controversy_floor_7` two-prompt averages improved mean quality `7.83 → 8.25`, min quality `7.00 → 8.00`, disagreement `0.33 → 0.75`; lexical overlap also rose `0.056 → 0.074`; meta-preamble stayed `0.0`. Promising but not enough for default changes because both expanded runs degraded. - 2026-06-05 updated-model L3 smoke completed (`todos/013`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`) after Pi exposed `pi/kimi-coding/kimi-for-coding` (Kimi K2.6 for coding) and `pi/minimax/MiniMax-M3`. Single-model smoke calls for both worked. A two-model product baseline (`prompt-variants off`) completed clean with `total_calls: 8`, `degraded: false`, `controversy_floor_7` mean/min quality `7.50/7.00`, lexical overlap `0.073`. A two-model prompt-reframing run completed degraded with `total_calls: 25/26`, final candidates `5`, and one MiniMax M3 round-2 proposal timeout on the legal-scrutiny variant after 900s; `controversy_floor_7` mean/min quality `8.33/8.00`, lexical overlap `0.080`, meta-preamble `0.0`. Because two-model runs have only one evaluator per candidate, disagreement/controversy is not meaningful. A four-model updated sample (Codex + GLM + Kimi-for-coding + MiniMax M3) was stopped after ~14 minutes while still in the first baseline run; partial artifacts showed round-1 progress, so treat it as a budget/runtime caution rather than a correctness failure. Keep production defaults unchanged. - 2026-06-04 Pi stream parsing completed (`todos/026`, plan `docs/plans/2026-06-04-002-fix-stream-parse-pi-json-events-plan.md`) after L3 prompt-reframing smoke re-exposed the 64MB Pi stdout cap. Added `process::spawn_cli_stream_lines()` with stderr draining, timeout/idle-timeout handling, and bounded error previews; `PiProvider` now feeds JSONL lines into a stateful parser shared with `extract_pi_response()`. The pre-fix 3-model L3 smoke degraded with Codex `ResponseTooLarge` (~64MB) and invalid GLM eval scores. The post-fix rerun completed `degraded: false`, `evaluation_status: peer_evaluated`, `total_calls: 75`, no provider failures, empty stderr. Smoke report: `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`. Verified with `cargo fmt --all -- --check`, `cargo test -p tundish_providers pi -q`, streaming process test, `cargo clippy -p tundish_providers --all-targets -- -D warnings`, `cargo test --workspace --no-fail-fast`, and `cargo clippy --workspace --all-targets -- -D warnings`. @@ -98,6 +99,6 @@ Recommended order: 1. If continuing Buildkite migration, review PR #39 and either trigger a real Buildkite run against `ci-linux-arm64-rust-bazel` or update the Buildkite pipeline settings to upload `.buildkite/pipeline.yml` from the repo so PR pipeline changes are exercised. 2. Start from clean `main` and read this handoff plus the valid baseline in `docs/brainstorms/2026-05-23-brainstorm-smoke-baseline.md`. -3. If continuing brainstorm strategy work, read `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`, and `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`; then either run 2-4 more L3 prompt-reframing prompts with the Codex/GLM/Kimi-for-coding panel or triage GLM invalid evaluation scores on expanded prompts. Do not launch a full MiniMax M3-heavy suite without explicit runtime/output budget controls. +3. If continuing brainstorm strategy work, read `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`, and `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`; then either run 2-4 more L3 prompt-reframing prompts with the Codex/GLM/Kimi-for-coding panel to live-validate the 2026-06-11 evaluation parser hardening, or continue deeper GLM invalid-score triage if failures persist. Do not launch a full MiniMax M3-heavy suite without explicit runtime/output budget controls. 4. For future Pi-backed benchmark runs, use `--max-concurrent 1` unless Pi config locking is fixed; for OpenCode-backed models use `--max-concurrent 1` and `--idle-timeout 480` until `todos/022` is fixed. 5. Do not implement Open Collider-style domain collisions before benchmark budget constraints are explicit; if moving to L3, start with prompt-reframing expansion from `todos/018`. diff --git a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md index 0a354c2..443ac10 100644 --- a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md +++ b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md @@ -16,6 +16,7 @@ todo: 013-brainstorm-strategy-benchmarks **Addendum:** 2026-06-01 — Completed first-pass qualitative L2 panel review; see `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`. **Addendum:** 2026-06-05 — Ran an updated-model L3 smoke with `pi/kimi-coding/kimi-for-coding:off` and `pi/minimax/MiniMax-M3:off`; see `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`. **Addendum:** 2026-06-09 — Ran a two-prompt three-model L3 comparison (`off` vs `per-model`) with Codex, GLM, and Kimi-for-coding; see `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. +**Addendum:** 2026-06-11 — Completed a verified brainstorm evaluation parser hardening pass for GLM-style invalid-score failures: score parsing now accepts scaled score text, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores. Targeted parser/brainstorm tests and `refinery_core` clippy passed. ## Context diff --git a/todos/013-brainstorm-strategy-benchmarks.md b/todos/013-brainstorm-strategy-benchmarks.md index 1cb572b..8fa79f1 100644 --- a/todos/013-brainstorm-strategy-benchmarks.md +++ b/todos/013-brainstorm-strategy-benchmarks.md @@ -4,7 +4,7 @@ priority: low milestone: v0.4 depends_on: 004-verb-brainstorm status: in_progress -updated: 2026-06-09 +updated: 2026-06-11 --- # Benchmark: Brainstorm Iteration and Selection Strategies @@ -104,7 +104,9 @@ A first-pass qualitative review over the generated blind panel review pack is co Latest L3 three-model sample is documented in `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. It compared `--prompt-variants off` vs `per-model` on product and technical prompts with Codex, GLM, and Kimi-for-coding. Per-model improved two-prompt `controversy_floor_7` average quality floor (`7.00` → `8.00`) and disagreement (`0.33` → `0.75`), but both per-model runs degraded due to evaluation issues (GLM invalid eval scores; one Codex SSE header timeout), so it cannot support default changes. -Next concrete step: either run a human/calibrated model-judge pass over the L2/L3 panel findings, run 2-4 more L3 prompts with the same three-model panel, or first harden/triage GLM invalid evaluation scores on expanded prompts. For L3, use `score-only` as the baseline, treat `own-reviews` as optional, and avoid launching a full 4-model × 6-prompt suite with MiniMax M3 until latency/output budget controls are explicit. +A verified parser hardening pass for recoverable GLM-style invalid evaluation scores completed on 2026-06-11. Brainstorm evaluation parsing now accepts scaled score text, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores while rejecting incomplete dimension sets. Verified with targeted parser tests, `cargo test -p refinery_core brainstorm -q`, and `cargo clippy -p refinery_core --all-targets -- -D warnings`. + +Next concrete step: either run a human/calibrated model-judge pass over the L2/L3 panel findings, run 2-4 more L3 prompts with the same three-model panel to validate the parser hardening in live Pi-backed expanded runs, or continue deeper GLM triage if invalid scores persist. For L3, use `score-only` as the baseline, treat `own-reviews` as optional, and avoid launching a full 4-model × 6-prompt suite with MiniMax M3 until latency/output budget controls are explicit. ## References From ca0242acfd81940116620d1384f32407449da026 Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:35:08 +0000 Subject: [PATCH 2/5] docs: add L3 parser validation runbook --- docs/HANDOFF.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 39986ef..175dce8 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -93,6 +93,61 @@ Triage pattern: fix P1/P2 with code, create TODOs for P3/nitpicks, reply to ever - Brainstorm divergence discussion captured in `docs/plans/2026-03-31-001-feat-brainstorm-verb-plan.md` addendum and `todos/018-brainstorm-divergence-expansion.md`: v0 preserves divergence through score-only controversial selection; future work should inject divergence via prompt reframing (`n(n+1)` lineages) and optional Open Collider-style domain collisions (`n(1+p)d` lineages). - `docs/solutions/` has solution docs covering Ctrl+C/SIGINT, provider quirks, prompt injection, tiebreaking, etc. +## Next L3 Validation Runbook + +Purpose: live-validate commit `dc805a7` (`fix: harden brainstorm evaluation score parsing`) against the GLM invalid-evaluation failures observed in the 2026-06-09 expanded L3 runs. + +Suggested artifact root: + +```text +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/ +``` + +For each selected prompt, run a paired baseline and expanded prompt-reframing run with the same three-model panel: + +```sh +ROOT=target/brainstorm-benchmark-2026-06-11-l3-parser-validation +PROMPT='' +mkdir -p "$ROOT/logs" + +cargo run -q -p refinery_cli -- brainstorm "$PROMPT" \ + --models pi/openai-codex/gpt-5.4:off,pi/zai/glm-5.1:off,pi/kimi-coding/kimi-for-coding:off \ + --max-rounds 2 \ + --panel-size 3 \ + --quality-floor 7.0 \ + --iteration-strategy score-only \ + --prompt-variants off \ + --output-dir "$ROOT/off/" \ + --output-format json \ + --verbose \ + --idle-timeout 480 \ + --timeout 1800 \ + --max-concurrent 1 > "$ROOT/logs/-off.json" 2> "$ROOT/logs/-off.stderr.log" + +cargo run -q -p refinery_cli -- brainstorm "$PROMPT" \ + --models pi/openai-codex/gpt-5.4:off,pi/zai/glm-5.1:off,pi/kimi-coding/kimi-for-coding:off \ + --max-rounds 2 \ + --panel-size 3 \ + --quality-floor 7.0 \ + --iteration-strategy score-only \ + --prompt-variants per-model \ + --output-dir "$ROOT/per-model/" \ + --output-format json \ + --verbose \ + --idle-timeout 480 \ + --timeout 1800 \ + --max-concurrent 1 > "$ROOT/logs/-per-model.json" 2> "$ROOT/logs/-per-model.stderr.log" +``` + +Record each generated run directory in `$ROOT/logs/run-dirs.txt`, then analyze with: + +```sh +cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat "$ROOT/logs/run-dirs.txt") --output-format text > "$ROOT/logs/l3-parser-validation-analysis.txt" +cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat "$ROOT/logs/run-dirs.txt") --output-format json > "$ROOT/logs/l3-parser-validation-analysis.json" +``` + +Decision check: inspect every `provider-failures.json`. If GLM still reports `provider returned an invalid brainstorm evaluation score`, preserve/capture the raw invalid evaluation response in the next code pass before further parser guessing. If expanded runs are non-degraded, continue 2-4 total prompts and compare against `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. + ## Next Clean Session Recommended order: From 1ce279742a7759e976576604b22df10998b12225 Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Fri, 12 Jun 2026 11:24:46 +0000 Subject: [PATCH 3/5] fix: capture invalid brainstorm eval responses --- .../refinery_cli/src/commands/brainstorm.rs | 3 + crates/refinery_core/src/brainstorm.rs | 67 ++++++++++++- docs/HANDOFF.md | 11 ++- ...6-06-12-brainstorm-l3-parser-validation.md | 99 +++++++++++++++++++ ...rch-brainstorm-strategy-benchmarks-plan.md | 3 +- todos/013-brainstorm-strategy-benchmarks.md | 4 +- 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md diff --git a/crates/refinery_cli/src/commands/brainstorm.rs b/crates/refinery_cli/src/commands/brainstorm.rs index 4eb37c4..e56bd35 100644 --- a/crates/refinery_cli/src/commands/brainstorm.rs +++ b/crates/refinery_cli/src/commands/brainstorm.rs @@ -82,6 +82,8 @@ struct ProviderFailureOutput { #[serde(skip_serializing_if = "Option::is_none")] target_model_id: Option, message: String, + #[serde(skip_serializing_if = "Option::is_none")] + response_preview: Option, } #[derive(Serialize)] @@ -385,6 +387,7 @@ fn provider_failure_output(failure: &BrainstormProviderFailure) -> ProviderFailu model_id: failure.model_id.to_string(), target_model_id: failure.target_model_id.as_ref().map(ToString::to_string), message: failure.message.clone(), + response_preview: failure.response_preview.clone(), } } diff --git a/crates/refinery_core/src/brainstorm.rs b/crates/refinery_core/src/brainstorm.rs index 15c32e0..ccb534a 100644 --- a/crates/refinery_core/src/brainstorm.rs +++ b/crates/refinery_core/src/brainstorm.rs @@ -142,6 +142,7 @@ pub struct BrainstormProviderFailure { pub model_id: ModelId, pub target_model_id: Option, pub message: String, + pub response_preview: Option, } /// Whether peer evaluation produced usable scores for the brainstorm panel. @@ -278,6 +279,15 @@ struct ParsedBrainstormEvaluation { rationale: String, } +const INVALID_RESPONSE_PREVIEW_CHARS: usize = 2_000; + +fn invalid_response_preview(response: &str) -> String { + response + .chars() + .take(INVALID_RESPONSE_PREVIEW_CHARS) + .collect() +} + fn parse_prompt_variant_response(response: &str) -> Option { let parsed = prompts::extract_json(response) .and_then(|json| serde_json::from_str::(json).ok()) @@ -656,6 +666,7 @@ async fn generate_prompt_variants( target_model_id: None, message: "provider returned an invalid brainstorm prompt variant" .to_string(), + response_preview: Some(invalid_response_preview(&response)), }); } } @@ -665,6 +676,7 @@ async fn generate_prompt_variants( model_id, target_model_id: None, message: err.to_string(), + response_preview: None, }), Ok((model_id, Err(_))) => { let err = ProviderError::Timeout { @@ -677,6 +689,7 @@ async fn generate_prompt_variants( model_id, target_model_id: None, message: err.to_string(), + response_preview: None, }); } Err(err) => provider_failures.push(BrainstormProviderFailure { @@ -685,6 +698,7 @@ async fn generate_prompt_variants( model_id: join_error_model_id(), target_model_id: None, message: err.to_string(), + response_preview: None, }), } } @@ -854,6 +868,7 @@ pub async fn run( model_id, target_model_id: None, message: "provider returned an empty proposal".to_string(), + response_preview: None, }); } else { round_proposals.insert(model_id, answer); @@ -867,6 +882,7 @@ pub async fn run( model_id, target_model_id: None, message: err.to_string(), + response_preview: None, }); } Ok((model_id, Err(_))) => { @@ -881,6 +897,7 @@ pub async fn run( model_id, target_model_id: None, message: err.to_string(), + response_preview: None, }); } Err(err) => { @@ -891,6 +908,7 @@ pub async fn run( model_id: join_error_model_id(), target_model_id: None, message: err.to_string(), + response_preview: None, }); } } @@ -1050,6 +1068,7 @@ pub async fn run( target_model_id: Some(to), message: "provider returned an invalid brainstorm evaluation score" .to_string(), + response_preview: Some(invalid_response_preview(&response)), }); } } @@ -1062,6 +1081,7 @@ pub async fn run( model_id: from, target_model_id: Some(to), message: err.to_string(), + response_preview: None, }); } Ok((from, to, Err(_))) => { @@ -1077,6 +1097,7 @@ pub async fn run( model_id: from, target_model_id: Some(to), message: err.to_string(), + response_preview: None, }); } Err(err) => { @@ -1088,6 +1109,7 @@ pub async fn run( model_id: join_error_model_id(), target_model_id: None, message: err.to_string(), + response_preview: None, }); } } @@ -1334,13 +1356,18 @@ fn save_provider_failures( let failures_json: Vec = failures .iter() .map(|failure| { - serde_json::json!({ + let mut failure_json = serde_json::json!({ "round": failure.round, "phase": failure.phase.to_string(), "model_id": failure.model_id.to_string(), "target_model_id": failure.target_model_id.as_ref().map(ToString::to_string), "message": &failure.message, - }) + }); + if let Some(response_preview) = &failure.response_preview { + failure_json["response_preview"] = + serde_json::Value::String(response_preview.clone()); + } + failure_json }) .collect(); std::fs::write( @@ -1401,6 +1428,14 @@ mod tests { } } + #[test] + fn invalid_response_preview_is_bounded() { + let response = "x".repeat(INVALID_RESPONSE_PREVIEW_CHARS + 10); + let preview = invalid_response_preview(&response); + + assert_eq!(preview.len(), INVALID_RESPONSE_PREVIEW_CHARS); + } + #[test] fn parse_brainstorm_evaluation_accepts_string_score() { let parsed = @@ -1843,6 +1878,34 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn invalid_evaluation_failure_captures_response_preview() { + let valid = EchoProvider::new("test/valid"); + valid.queue_response(r#"{"answer": "valid answer"}"#.to_string()); + valid.queue_response(eval_json(8)); + + let invalid = EchoProvider::new("test/invalid"); + invalid.queue_response(r#"{"answer": "invalid answer"}"#.to_string()); + invalid.queue_response(r#"{"rationale":"not numeric","score":"excellent"}"#.to_string()); + + let providers: Vec> = vec![Arc::new(valid), Arc::new(invalid)]; + let config = default_config(1, 2); + let result = run(&providers, "test prompt", &config).await.unwrap(); + + let failure = result + .provider_failures + .iter() + .find(|failure| failure.model_id == ModelId::new("test/invalid")) + .expect("invalid evaluator should be captured as a provider failure"); + + assert_eq!(failure.phase, Phase::Evaluate); + assert_eq!(failure.target_model_id, Some(ModelId::new("test/valid"))); + assert_eq!( + failure.response_preview.as_deref(), + Some(r#"{"rationale":"not numeric","score":"excellent"}"#) + ); + } + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn controversial_answer_ranks_higher() { // 3 models, 1 round. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 175dce8..3fbef7c 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -2,7 +2,7 @@ Current state of the project and active work. Read this at session start. Update before compaction or at natural breakpoints. -**Last updated:** 2026-06-11 +**Last updated:** 2026-06-12 ## Project State @@ -45,7 +45,7 @@ See `memory/verb_architecture.md` for full taxonomy with consistent terminology. Check `todos/` for the full list. Key ones: -- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, benchmark-only iteration variants, L2 six-prompt variant suite, blind review pack, first-pass qualitative L2 panel review, hidden L3 prompt-reframing implementation, 3-model L3 smoke, updated-model 2-model L3 smoke, two-prompt 3-model L3 sample, and initial verified parser hardening for recoverable GLM-style invalid eval scores completed; next step is either live-validate the parser hardening with 2-4 more L3 prompts using Codex/GLM/Kimi-for-coding or continue deeper GLM triage if invalid eval scores persist; avoid full MiniMax M3 suites until runtime/output budget controls are explicit +- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, benchmark-only iteration variants, L2 six-prompt variant suite, blind review pack, first-pass qualitative L2 panel review, hidden L3 prompt-reframing implementation, 3-model L3 smoke, updated-model 2-model L3 smoke, two-prompt 3-model L3 sample, initial parser hardening for recoverable eval-score shapes, and one post-hardening L3 validation completed; next step is a small L3 rerun after bounded raw invalid-response previews are captured, to distinguish malformed/empty/provider output from genuinely unhandled score JSON; avoid full MiniMax M3 suites until runtime/output budget controls are explicit - **025** — optional brainstorm lineage-reference polish if softer phrases like "builds on..." feel too process-oriented in demos - **018** — brainstorm divergence expansion: first-stage prompt reframing implemented behind hidden `brainstorm --prompt-variants per-model`; next run L3 benchmarks and defer domain collisions - **021** — evaluate TOON (`toon-format/toon`) for prompt-facing artifact export / benchmark fixtures @@ -59,7 +59,8 @@ Triage pattern: fix P1/P2 with code, create TODOs for P3/nitpicks, reply to ever ## Recent Context -- 2026-06-11 GLM invalid-evaluation parser hardening completed (`todos/013`, plan `docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md`). `parse_brainstorm_evaluation_response()` now accepts recoverable score variants seen/plausible in expanded brainstorm evals: scaled score text like `"8 out of 10"`, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores. It rejects incomplete dimension sets even if extra numeric fields are present. Verified with `cargo fmt --all -- --check`, `cargo test -p refinery_core parse_brainstorm_evaluation -q`, `cargo test -p refinery_core brainstorm -q`, and `cargo clippy -p refinery_core --all-targets -- -D warnings`. Next useful validation is a live Pi-backed L3 run with Codex/GLM/Kimi-for-coding; if invalid eval scores persist, capture/preserve raw invalid responses for deeper GLM triage. +- 2026-06-12 post-`dc805a7` L3 parser validation completed (`docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md`). Ran one paired architecture prompt using Codex/GLM/Kimi-for-coding. Baseline `off` degraded with one GLM invalid eval score; expanded `per-model` degraded with three Kimi 429 overload/rate-limit failures and one Kimi invalid eval score. Analyzer used exactly the two fresh run dirs in `target/brainstorm-benchmark-2026-06-11-l3-parser-validation/logs/run-dirs.txt`. Conclusion: invalid evaluation summaries persist after parser hardening, but this does not prove another score-shape parsing gap. Added bounded raw response-preview capture for invalid brainstorm structured-response parse failures so the next rerun can distinguish malformed/empty/provider output from genuinely unhandled JSON shape. +- 2026-06-11 GLM invalid-evaluation parser hardening completed (`todos/013`, plan `docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md`). `parse_brainstorm_evaluation_response()` now accepts recoverable score variants seen/plausible in expanded brainstorm evals: scaled score text like `"8 out of 10"`, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores. It rejects incomplete dimension sets even if extra numeric fields are present. Verified with `cargo fmt --all -- --check`, `cargo test -p refinery_core parse_brainstorm_evaluation -q`, `cargo test -p refinery_core brainstorm -q`, and `cargo clippy -p refinery_core --all-targets -- -D warnings`. - 2026-06-09 three-model L3 sample completed (`todos/013`, `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`) after PR #44 merged. Compared `--prompt-variants off` vs `per-model` on product and technical prompts using `pi/openai-codex/gpt-5.4:off`, `pi/zai/glm-5.1:off`, and `pi/kimi-coding/kimi-for-coding:off`, serial with `--max-concurrent 1`. Baseline runs completed clean (`18` calls each, ~7-8m). Per-model runs completed with full 12-candidate final sets but degraded evaluation status (`75` calls each, ~30-38m): product had one GLM invalid eval score; technical had one Codex SSE response-header timeout and one GLM invalid eval score. `controversy_floor_7` two-prompt averages improved mean quality `7.83 → 8.25`, min quality `7.00 → 8.00`, disagreement `0.33 → 0.75`; lexical overlap also rose `0.056 → 0.074`; meta-preamble stayed `0.0`. Promising but not enough for default changes because both expanded runs degraded. - 2026-06-05 updated-model L3 smoke completed (`todos/013`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`) after Pi exposed `pi/kimi-coding/kimi-for-coding` (Kimi K2.6 for coding) and `pi/minimax/MiniMax-M3`. Single-model smoke calls for both worked. A two-model product baseline (`prompt-variants off`) completed clean with `total_calls: 8`, `degraded: false`, `controversy_floor_7` mean/min quality `7.50/7.00`, lexical overlap `0.073`. A two-model prompt-reframing run completed degraded with `total_calls: 25/26`, final candidates `5`, and one MiniMax M3 round-2 proposal timeout on the legal-scrutiny variant after 900s; `controversy_floor_7` mean/min quality `8.33/8.00`, lexical overlap `0.080`, meta-preamble `0.0`. Because two-model runs have only one evaluator per candidate, disagreement/controversy is not meaningful. A four-model updated sample (Codex + GLM + Kimi-for-coding + MiniMax M3) was stopped after ~14 minutes while still in the first baseline run; partial artifacts showed round-1 progress, so treat it as a budget/runtime caution rather than a correctness failure. Keep production defaults unchanged. - 2026-06-04 Pi stream parsing completed (`todos/026`, plan `docs/plans/2026-06-04-002-fix-stream-parse-pi-json-events-plan.md`) after L3 prompt-reframing smoke re-exposed the 64MB Pi stdout cap. Added `process::spawn_cli_stream_lines()` with stderr draining, timeout/idle-timeout handling, and bounded error previews; `PiProvider` now feeds JSONL lines into a stateful parser shared with `extract_pi_response()`. The pre-fix 3-model L3 smoke degraded with Codex `ResponseTooLarge` (~64MB) and invalid GLM eval scores. The post-fix rerun completed `degraded: false`, `evaluation_status: peer_evaluated`, `total_calls: 75`, no provider failures, empty stderr. Smoke report: `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`. Verified with `cargo fmt --all -- --check`, `cargo test -p tundish_providers pi -q`, streaming process test, `cargo clippy -p tundish_providers --all-targets -- -D warnings`, `cargo test --workspace --no-fail-fast`, and `cargo clippy --workspace --all-targets -- -D warnings`. @@ -146,7 +147,7 @@ cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat "$ROOT/logs/run-dirs. cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat "$ROOT/logs/run-dirs.txt") --output-format json > "$ROOT/logs/l3-parser-validation-analysis.json" ``` -Decision check: inspect every `provider-failures.json`. If GLM still reports `provider returned an invalid brainstorm evaluation score`, preserve/capture the raw invalid evaluation response in the next code pass before further parser guessing. If expanded runs are non-degraded, continue 2-4 total prompts and compare against `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. +Decision check: inspect every `provider-failures.json`. Invalid structured-response parse failures should now include `response_preview`; use that preview to decide whether the issue is malformed/empty/provider output or genuinely unhandled score JSON before further parser guessing. If expanded runs are non-degraded, continue 2-4 total prompts and compare against `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. ## Next Clean Session @@ -154,6 +155,6 @@ Recommended order: 1. If continuing Buildkite migration, review PR #39 and either trigger a real Buildkite run against `ci-linux-arm64-rust-bazel` or update the Buildkite pipeline settings to upload `.buildkite/pipeline.yml` from the repo so PR pipeline changes are exercised. 2. Start from clean `main` and read this handoff plus the valid baseline in `docs/brainstorms/2026-05-23-brainstorm-smoke-baseline.md`. -3. If continuing brainstorm strategy work, read `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`, and `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`; then either run 2-4 more L3 prompt-reframing prompts with the Codex/GLM/Kimi-for-coding panel to live-validate the 2026-06-11 evaluation parser hardening, or continue deeper GLM invalid-score triage if failures persist. Do not launch a full MiniMax M3-heavy suite without explicit runtime/output budget controls. +3. If continuing brainstorm strategy work, read `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, `docs/brainstorms/2026-06-04-brainstorm-l3-prompt-reframing-smoke.md`, `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`, `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`, and `docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md`; then rerun a small L3 prompt-reframing validation with the Codex/GLM/Kimi-for-coding panel and inspect new `response_preview` fields for any invalid structured-response parse failures. Do not launch a full MiniMax M3-heavy suite without explicit runtime/output budget controls. 4. For future Pi-backed benchmark runs, use `--max-concurrent 1` unless Pi config locking is fixed; for OpenCode-backed models use `--max-concurrent 1` and `--idle-timeout 480` until `todos/022` is fixed. 5. Do not implement Open Collider-style domain collisions before benchmark budget constraints are explicit; if moving to L3, start with prompt-reframing expansion from `todos/018`. diff --git a/docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md b/docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md new file mode 100644 index 0000000..d8f41df --- /dev/null +++ b/docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md @@ -0,0 +1,99 @@ +--- +date: 2026-06-12 +topic: brainstorm-l3-parser-validation +todo: 013-brainstorm-strategy-benchmarks +plan: 2026-05-23-001-research-brainstorm-strategy-benchmarks-plan +related_commit: dc805a7 +--- + +# Brainstorm L3 Parser Validation + +## Summary + +Ran one paired L3 validation prompt after commit `dc805a7` (`fix: harden brainstorm evaluation score parsing`) to see whether the parser hardening eliminated invalid-evaluation degradation from the Codex/GLM/Kimi-for-coding panel. + +Result: both runs still degraded. The baseline (`--prompt-variants off`) still had one GLM invalid brainstorm evaluation score. The expanded run (`--prompt-variants per-model`) had Kimi overload/rate-limit failures and one Kimi invalid brainstorm evaluation score. This means the next useful triage step is to preserve bounded raw invalid evaluation responses in provider failure records rather than continuing to guess parser variants from failure summaries alone. + +## Prompt + +Architecture/design prompt: + +```text +Design a plugin system for local AI tools with strong sandboxing, explicit user consent, and useful extension ergonomics. Generate unconventional but practical architecture ideas. +``` + +## Models + +```text +pi/openai-codex/gpt-5.4:off +pi/zai/glm-5.1:off +pi/kimi-coding/kimi-for-coding:off +``` + +## Common Settings + +```text +--max-rounds 2 +--panel-size 3 +--quality-floor 7.0 +--iteration-strategy score-only +--idle-timeout 480 +--timeout 1800 +--max-concurrent 1 +``` + +## Artifacts + +Root: + +```text +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/ +``` + +Run dirs: + +```text +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/off/architecture-plugin-sandbox/20260612-101507_design-a-plugin-system-for-local-ai-tool_f92e +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/per-model/architecture-plugin-sandbox/20260612-103202_design-a-plugin-system-for-local-ai-tool_ae1d +``` + +Analyzer outputs: + +```text +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/logs/l3-parser-validation-analysis.txt +target/brainstorm-benchmark-2026-06-11-l3-parser-validation/logs/l3-parser-validation-analysis.json +``` + +`logs/run-dirs.txt` contained exactly the two run dirs above; the analyzer JSON also referenced exactly those two paths, so no stale artifacts were included in this result. + +## Run Results + +| Prompt variants | Status | Eval status | Calls | Elapsed | Provider failures | +|---|---|---|---:|---:|---| +| `off` | `degraded` | `partial` | 18 | ~16.7m | GLM invalid eval score | +| `per-model` | `degraded` | `partial` | 75 | ~43.5m | 3 Kimi 429 overloads; 1 Kimi invalid eval score | + +Failure details: + +- Baseline round 2: `pi/zai/glm-5.1:off` returned an invalid brainstorm evaluation score while evaluating `pi/openai-codex/gpt-5.4:off`. +- Expanded round 1: `pi/kimi-coding/kimi-for-coding:off` returned three provider errors: `429 {"error":{"type":"rate_limit_error","message":"The engine is currently overloaded, please try again later"}}`. +- Expanded round 2: `pi/kimi-coding/kimi-for-coding:off` returned an invalid brainstorm evaluation score while evaluating `pi/openai-codex_gpt-5.4:off+variant-1`. + +## Production-Selector Metrics + +`controversy_floor_7` view: + +| Prompt variants | Mean quality | Min quality | Disagreement | Lexical overlap | Meta preamble rate | +|---|---:|---:|---:|---:|---:| +| `off` | 7.67 | 7.00 | 0.33 | 0.088 | 0.00 | +| `per-model` | 7.83 | 7.50 | 0.83 | 0.081 | 0.00 | + +The expanded run again improved disagreement and slightly improved the selected panel quality floor, but degradation prevents drawing a default-change conclusion. + +## Decision + +Do not run more L3 prompts solely to validate parser hardening without better failure evidence. Invalid evaluation summaries still occur after `dc805a7`, and these artifacts do not contain the raw invalid response text from this run. + +Follow-up implemented in the same session: bounded raw response preview capture was added to `BrainstormProviderFailure` for invalid structured-response parse failures and exposed in CLI JSON plus `provider-failures.json`. + +Next step: rerun a small live validation so any remaining invalid-score failure includes enough evidence to distinguish malformed/empty/provider output from genuinely unhandled score JSON. diff --git a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md index 443ac10..2c6387d 100644 --- a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md +++ b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md @@ -17,6 +17,7 @@ todo: 013-brainstorm-strategy-benchmarks **Addendum:** 2026-06-05 — Ran an updated-model L3 smoke with `pi/kimi-coding/kimi-for-coding:off` and `pi/minimax/MiniMax-M3:off`; see `docs/brainstorms/2026-06-05-brainstorm-l3-updated-model-smoke.md`. **Addendum:** 2026-06-09 — Ran a two-prompt three-model L3 comparison (`off` vs `per-model`) with Codex, GLM, and Kimi-for-coding; see `docs/brainstorms/2026-06-09-brainstorm-l3-three-model-sample.md`. **Addendum:** 2026-06-11 — Completed a verified brainstorm evaluation parser hardening pass for GLM-style invalid-score failures: score parsing now accepts scaled score text, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores. Targeted parser/brainstorm tests and `refinery_core` clippy passed. +**Addendum:** 2026-06-12 — Live L3 parser validation still produced invalid evaluation summaries after `dc805a7`; see `docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md`. Added bounded raw response-preview capture for invalid brainstorm structured-response parse failures so the next degraded run preserves triage evidence. ## Context @@ -179,7 +180,7 @@ Result: `score-only` remained strongest on useful diversity/non-overlap; `full-v ## Next Implementation Step -Continue `todos/013` with either a human/calibrated model-judge pass over `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, 2-4 more L3 prompts with the Codex/GLM/Kimi-for-coding panel, or hardening/triage for GLM invalid evaluation scores on expanded prompt-reframing runs. The 2026-06-05 updated-model smoke showed Kimi-for-coding and MiniMax M3 are available through Pi, but MiniMax M3 can dominate runtime and timed out on one expanded product-prompt lineage; do not launch a full 4-model × 6-prompt L3 suite until latency/output budget controls are explicit. The 2026-06-09 three-model sample showed promising quality-floor/disagreement gains for `per-model`, but both expanded runs degraded due to evaluator failures. Do not change the production default based on the first-pass L2 review or small L3 samples alone. +Continue `todos/013` with either a human/calibrated model-judge pass over `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`, or a small L3 rerun with the Codex/GLM/Kimi-for-coding panel after raw invalid-response previews are available. The 2026-06-05 updated-model smoke showed Kimi-for-coding and MiniMax M3 are available through Pi, but MiniMax M3 can dominate runtime and timed out on one expanded product-prompt lineage; do not launch a full 4-model × 6-prompt L3 suite until latency/output budget controls are explicit. The 2026-06-09 three-model sample showed promising quality-floor/disagreement gains for `per-model`, but both expanded runs degraded due to evaluator failures. The 2026-06-12 parser validation confirmed invalid evaluation summaries persist after parser hardening, so preserve raw invalid responses before further parser/prompt guessing. Do not change the production default based on the first-pass L2 review or small L3 samples alone. ## Verification diff --git a/todos/013-brainstorm-strategy-benchmarks.md b/todos/013-brainstorm-strategy-benchmarks.md index 8fa79f1..9a19cd5 100644 --- a/todos/013-brainstorm-strategy-benchmarks.md +++ b/todos/013-brainstorm-strategy-benchmarks.md @@ -106,7 +106,9 @@ Latest L3 three-model sample is documented in `docs/brainstorms/2026-06-09-brain A verified parser hardening pass for recoverable GLM-style invalid evaluation scores completed on 2026-06-11. Brainstorm evaluation parsing now accepts scaled score text, nested score objects, `overall_score`, and a missing-overall fallback to the four required dimension scores while rejecting incomplete dimension sets. Verified with targeted parser tests, `cargo test -p refinery_core brainstorm -q`, and `cargo clippy -p refinery_core --all-targets -- -D warnings`. -Next concrete step: either run a human/calibrated model-judge pass over the L2/L3 panel findings, run 2-4 more L3 prompts with the same three-model panel to validate the parser hardening in live Pi-backed expanded runs, or continue deeper GLM triage if invalid scores persist. For L3, use `score-only` as the baseline, treat `own-reviews` as optional, and avoid launching a full 4-model × 6-prompt suite with MiniMax M3 until latency/output budget controls are explicit. +A live post-hardening L3 validation on 2026-06-12 is documented in `docs/brainstorms/2026-06-12-brainstorm-l3-parser-validation.md`. It still degraded: baseline had one GLM invalid eval score, while expanded prompt-reframing had Kimi overload/rate-limit failures plus one Kimi invalid eval score. This does not prove a remaining score-shape parser gap; the next code step is evidence capture to distinguish malformed/empty/provider output from genuinely unhandled score JSON. + +Next concrete step: rerun a small L3 validation and inspect bounded `response_preview` fields for any invalid structured-response parse failures, or run a human/calibrated model-judge pass over the L2/L3 panel findings. For L3, use `score-only` as the baseline, treat `own-reviews` as optional, and avoid launching a full 4-model × 6-prompt suite with MiniMax M3 until latency/output budget controls are explicit. ## References From 7243d0d8002d3aa137bdcb519b16191adb323a0e Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Sun, 14 Jun 2026 18:53:01 +0000 Subject: [PATCH 4/5] test: verify invalid eval preview artifacts --- crates/refinery_core/src/brainstorm.rs | 35 ++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/refinery_core/src/brainstorm.rs b/crates/refinery_core/src/brainstorm.rs index ccb534a..757f00d 100644 --- a/crates/refinery_core/src/brainstorm.rs +++ b/crates/refinery_core/src/brainstorm.rs @@ -1906,6 +1906,41 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn invalid_evaluation_failure_artifact_includes_response_preview() { + let valid = EchoProvider::new("test/valid"); + valid.queue_response(r#"{"answer": "valid answer"}"#.to_string()); + valid.queue_response(eval_json(8)); + + let invalid_response = r#"{"rationale":"not numeric","score":"excellent"}"#; + let invalid = EchoProvider::new("test/invalid"); + invalid.queue_response(r#"{"answer": "invalid answer"}"#.to_string()); + invalid.queue_response(invalid_response.to_string()); + + let output_dir = std::env::temp_dir().join(format!( + "refinery-invalid-preview-{}-{}", + std::process::id(), + rand::random::() + )); + let _ = std::fs::remove_dir_all(&output_dir); + + let providers: Vec> = vec![Arc::new(valid), Arc::new(invalid)]; + let mut config = default_config(1, 2); + config.output_dir = Some(output_dir.clone()); + run(&providers, "test prompt", &config).await.unwrap(); + + let failures_path = output_dir.join("provider-failures.json"); + let failures: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&failures_path) + .expect("provider failures artifact should be written"), + ) + .expect("provider failures artifact should be valid JSON"); + + assert_eq!(failures[0]["response_preview"], invalid_response); + + let _ = std::fs::remove_dir_all(output_dir); + } + #[tokio::test(flavor = "current_thread", start_paused = true)] async fn controversial_answer_ranks_higher() { // 3 models, 1 round. From 5d51dce4f348ab25b1c62294b4ca362f1882e4db Mon Sep 17 00:00:00 2001 From: El-Fitz <8971906+El-Fitz@users.noreply.github.com> Date: Mon, 15 Jun 2026 16:38:36 +0000 Subject: [PATCH 5/5] fix: handle labeled brainstorm score text --- crates/refinery_core/src/brainstorm.rs | 87 ++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/crates/refinery_core/src/brainstorm.rs b/crates/refinery_core/src/brainstorm.rs index 757f00d..eb68256 100644 --- a/crates/refinery_core/src/brainstorm.rs +++ b/crates/refinery_core/src/brainstorm.rs @@ -345,6 +345,16 @@ fn parse_score_text(text: &str) -> Option { let lower = trimmed.to_ascii_lowercase(); if let Some(score_idx) = lower.rfind("score") { + let score_tail_start = score_idx + "score".len(); + let score_tail = &trimmed[score_tail_start..]; + if score_tail.contains(':') { + for segment in score_tail.split(':').skip(1) { + if let Some(score) = parse_score_text_segment(segment) { + return Some(score); + } + } + } + if let Some((start, end)) = number_spans .iter() .copied() @@ -354,11 +364,8 @@ fn parse_score_text(text: &str) -> Option { } } - if let Some((start, end)) = number_spans.first().copied() { - let after_first = lower[end..].trim_start(); - if after_first.starts_with("out of") || after_first.starts_with("/10") { - return trimmed[start..end].parse::().ok(); - } + if let Some(score) = parse_score_text_segment(trimmed) { + return Some(score); } let (start, end) = number_spans.last().copied()?; @@ -366,6 +373,24 @@ fn parse_score_text(text: &str) -> Option { }) } +fn parse_score_text_segment(text: &str) -> Option { + let spans = number_spans(text); + let (first_start, first_end) = spans.first().copied()?; + let lower = text.to_ascii_lowercase(); + let after_first = lower[first_end..].trim_start(); + + if after_first.starts_with("out of") || after_first.starts_with('/') { + return text[first_start..first_end].parse::().ok(); + } + + if lower.contains("scale") && spans.len() > 1 { + return None; + } + + let (start, end) = spans.last().copied()?; + text[start..end].parse::().ok() +} + fn number_spans(text: &str) -> Vec<(usize, usize)> { let chars: Vec<(usize, char)> = text.char_indices().collect(); let mut spans = Vec::new(); @@ -1433,7 +1458,7 @@ mod tests { let response = "x".repeat(INVALID_RESPONSE_PREVIEW_CHARS + 10); let preview = invalid_response_preview(&response); - assert_eq!(preview.len(), INVALID_RESPONSE_PREVIEW_CHARS); + assert_eq!(preview.chars().count(), INVALID_RESPONSE_PREVIEW_CHARS); } #[test] @@ -1465,6 +1490,56 @@ mod tests { assert!((parsed.score - 8.0).abs() < f64::EPSILON); } + #[test] + fn parse_brainstorm_evaluation_accepts_spaced_slash_scale_without_using_denominator() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"8.5 / 10"}"#, + ) + .expect("spaced slash-scale score text should parse"); + + assert!((parsed.score - 8.5).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_uses_score_after_labeled_scale() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"Score (1-10): 8"}"#, + ) + .expect("score after labeled scale should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_uses_score_after_unparenthesized_labeled_scale() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"Score 1-10: 8"}"#, + ) + .expect("score after unparenthesized labeled scale should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_uses_score_after_labeled_scale_with_two_colons() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"score: on a 1-10 scale: 8"}"#, + ) + .expect("score after labeled scale with two colons should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + + #[test] + fn parse_brainstorm_evaluation_keeps_score_before_later_labeled_prose() { + let parsed = parse_brainstorm_evaluation_response( + r#"{"rationale":"good tension","score":"Score: 8 rationale: strong but risky"}"#, + ) + .expect("score before later labeled prose should parse"); + + assert!((parsed.score - 8.0).abs() < f64::EPSILON); + } + #[test] fn parse_brainstorm_evaluation_accepts_plus_signed_score_text() { let parsed = parse_brainstorm_evaluation_response(