From 651e6d1ce1ae200ecac894bf6298062121631978 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Wed, 26 Aug 2026 13:01:35 +0200 Subject: [PATCH 1/4] docs: research --- .scratch/scott-cc-comparison/domain-model.md | 88 ++++ .../01-research-mutflow-fork-branches.md | 52 +++ .../issues/02-research-comparison-matrix.md | 66 +++ .../issues/03-decide-prioritize-gaps.md | 53 +++ .../04-decide-upstream-recommendations.md | 59 +++ .scratch/scott-cc-comparison/map.md | 54 +++ .../research/02-comparison-matrix.md | 218 ++++++++++ .../01-research-redundant-test-detection.md | 35 ++ .../issues/02-research-auto-refactoring.md | 34 ++ .../03-research-execution-gap-reporting.md | 40 ++ .../04-decide-redundant-test-detection.md | 52 +++ .../issues/05-decide-auto-refactoring.md | 50 +++ .../06-decide-execution-gap-reporting.md | 54 +++ .scratch/scott-cc-implementation/map.md | 59 +++ .../research/01-redundant-test-detection.md | 388 ++++++++++++++++++ .../research/02-research-auto-refactoring.md | 366 +++++++++++++++++ .../03-research-execution-gap-reporting.md | 346 ++++++++++++++++ 17 files changed, 2014 insertions(+) create mode 100644 .scratch/scott-cc-comparison/domain-model.md create mode 100644 .scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md create mode 100644 .scratch/scott-cc-comparison/issues/02-research-comparison-matrix.md create mode 100644 .scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md create mode 100644 .scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md create mode 100644 .scratch/scott-cc-comparison/map.md create mode 100644 .scratch/scott-cc-comparison/research/02-comparison-matrix.md create mode 100644 .scratch/scott-cc-implementation/issues/01-research-redundant-test-detection.md create mode 100644 .scratch/scott-cc-implementation/issues/02-research-auto-refactoring.md create mode 100644 .scratch/scott-cc-implementation/issues/03-research-execution-gap-reporting.md create mode 100644 .scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md create mode 100644 .scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md create mode 100644 .scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md create mode 100644 .scratch/scott-cc-implementation/map.md create mode 100644 .scratch/scott-cc-implementation/research/01-redundant-test-detection.md create mode 100644 .scratch/scott-cc-implementation/research/02-research-auto-refactoring.md create mode 100644 .scratch/scott-cc-implementation/research/03-research-execution-gap-reporting.md diff --git a/.scratch/scott-cc-comparison/domain-model.md b/.scratch/scott-cc-comparison/domain-model.md new file mode 100644 index 0000000..7e41da4 --- /dev/null +++ b/.scratch/scott-cc-comparison/domain-model.md @@ -0,0 +1,88 @@ +# Domain Model: Mutation Testing Systems + +## Core Entities + +### Mutation Engine +The tool that generates mutations in source code. + +| Implementation | Scott-CC | OMP | +|---|---|---| +| Engine type | LLM-guided semantic mutation | `mutflow` — Kotlin compiler plugin (IR transformer) | +| Mutation generation | Agent reads source, decides what to mutate | Compile-time IR branch injection (`MutationRegistry.check()`) | +| Operator catalog | 5 strategies chosen per-file by LLM | Predefined operators (RelationalComparison, BooleanReturn, etc.) | + +### Mutation Strategy +A category of realistic bug pattern. + +| # | Strategy | Scott-CC | OMP/mutflow | Fork bridge | +|---|---|---|---|---| +| 1 | Boundary conditions | `>=` → `>`, `==`, `<=` | `RelationalComparisonOperator`, `ConstantBoundaryOperator` | ✅ | +| 2 | Return values | `return x` → `return None/""/` | `BooleanReturnOperator`, `NullableReturnOperator` | ✅ | +| 3 | Boolean logic | `and` → `or`, negate | `BooleanInversionOperator`, `EqualitySwapOperator`, `BooleanLogicOperator` | ✅ | +| 4 | Arithmetic operators | `*` → `/`, `+`, `-` | `ArithmeticOperator` | ✅ (+ IR truncate fix) | +| 5 | Exception types | `raise ValueError` → `TypeError` | NO operator | ✅ via `ExceptionTypeSwapOperator` fork | + +### Test Isolation +How mutations are kept separate from the main working tree. + +| Approach | Scott-CC | OMP | +|---|---|---| +| Mechanism | Git worktree per mutation | Compile-once meta-mutant (all variants compiled, one active per run) | +| Tradeoff | Full parallelization possible | Global synchronized lock serializes runs | +| Safety | Main tree never touched | Compile-time only; main tree untouched | + +### Test Executor +The component that runs tests against mutated code. + +| Aspect | Scott-CC | OMP | +|---|---|---| +| Parallelism | 15 parallel agents (Nx speedup) | One per test class; mutflow serializes mutations within | +| Execution | `pytest` / `npm test` in worktree | `./gradlew test` (JUnit 6 extension) | +| Multi-run model | Not needed (separate worktrees per mutation) | Baseline (run 0) + N mutation runs (run 1+) | + +### Quality Analyzer +The component that interprets test results to assess test quality. + +| Feature | Scott-CC (test-auditor) | OMP (test-auditor + Gradle task) | +|---|---|---| +| Mutation score | killed / total | killed / total | +| Quality bands | >80% Excellent, >60% Good, >40% Fair | >80% Excellent, >60% Good, >30% Fair, <30% Poor | +| Confidence | Not tracked | Low (<10), Medium (10-50), High (50+) | +| Zombie detection | Tests never failed across all mutations | Per-test-per-mutation matrix (all killer tests tracked via fork) | +| Redundant groups | Tests that always fail together (>5 in same group) | ❌ Not present | +| Over-mocked detection | >5 mocks per test | >3 mocks per test (MockK + Mockito) | +| Missing coverage | Surviving mutations → suggestions | Surviving mutations → recommendations | + +### Test Refactoring +The component that proposes or generates improved test code. + +| Aspect | Scott-CC (test-refactor-specialist) | OMP | +|---|---|---| +| Output | Production-ready refactored test code | Suggestions only (no auto-apply) | +| Actions | Consolidate, remove zombies, add edge cases, replace over-mocked | Same suggestions | +| User involvement | Approval before applying | Agent proposes, user applies manually | +| Auto-apply | ✅ With `--auto-approve` | ❌ | + +### Interface +How the user invokes the system. + +| Aspect | Scott-CC | OMP | +|---|---|---| +| Entry point | `/mutation-test` slash command | `/mutation-test` skill → `task` dispatch | +| Auto-detection | Natural language triggers ("mutation test", "zombie tests") | Not present (explicit path required) | +| Modes | `--quick` (5), standard (15), `--deep` (30+) | N/A (mutflow controls mutation count) | +| Setup | Not needed (install plugin) | `/mutation-test setup` (bootstraps .omp/ files + buildSrc) | +| External integration | Beads (issue tracking) | OMP task tool, Gradle | + +## ADRs Referenced +- ADR-001: Engine selection — mutflow for Kotlin (compile-once), LLM for Python/JS +- ADR-002: Isolation strategy — compile-once meta-mutant eliminates git worktrees +- ADR-003: Result format — typed Kotlin JSON module in buildSrc (vs console parsing) + +## Key Terminology +- **Zombie test**: A test that passes even when the code is mutated (doesn't catch bugs) +- **Mutation score**: % of mutations caught by the test suite +- **Survived mutation**: A mutation where all tests passed (not caught) +- **Killed mutation**: A mutation where at least one test failed (caught) +- **Test killer matrix**: Maps each test → list of mutation source locations it killed +- **Meta-mutant**: All mutation variants injected at compile time, with runtime selection diff --git a/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md b/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md new file mode 100644 index 0000000..e1355b7 --- /dev/null +++ b/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md @@ -0,0 +1,52 @@ +Type: research +Status: resolved (redundant — see Answer) +Blocked by: (none) + +## Question + +What are the 12 upstream changes in the `exception-swap` mutflow fork (trancee/mutflow-exception-swap), and which of them are relevant to bridging Scott-CC feature gaps in OMP? + +### Background + +The mutflow fork at `trancee/mutflow-exception-swap` has 12 branches: + +``` +exception-type-swap +feature/exception-type-swap +feature/zombie-detection +introduce-verification-mode-strict-lenient-and-disabled +introduce-optional-junit-config-flag-skip-cause-not-all-cases-covered +add-option-to-define-mutation-targets-via-gradle-config +add-pipeline +avoid-mutating-null-checks +double-arithmetic-ir-when-truncate-fix +hint-for-gradle-and-jooq-user +kotlin-native +optional-extra-cli-safe-guard-verification +update-versions +``` + +### Task + +1. Clone the fork and examine each branch (diff against master, commit history, commit messages). +2. For each branch, document: what feature/change it adds and a brief technical description. +3. Filter to branches **relevant to Scott-CC feature gaps**. +4. For each relevant branch, map it to the Scott-CC gap it closes. + +### Acceptance criteria + +- Complete catalog of all 12 branches with descriptions. +- Filtered subset of Scott-CC-relevant branches with gap-mapping. +- Findings captured in this issue's resolution. + +--- + +## Answer + +**R1 closed as redundant.** R2's comparison matrix (ResearchComparisonMatrix) already cataloged all 12 mutflow fork branches in [section 8: Mutflow Fork Branches That Bridge Gaps](research/02-comparison-matrix.md#8), including: + +- Branch name, commit hash, files changed, what each adds, Scott-CC gap bridged, and upstreamability assessment +- Full fork chain: `master → feature/exception-type-swap → feature/zombie-detection → avoid-mutating-null-checks → double-arithmetic-ir-when-truncate-fix → optional-extra-cli-safe-guard-verification → introduce-verification-mode-… → add-option-to-define-mutation-targets-via-gradle-config → add-pipeline → update-versions` +- Relevant branches identified: `exception-type-swap` (Strategy 5: Exception types), `zombie-detection` (multi-killer tracking), `double-arithmetic-ir-when-truncate-fix` (arithmetic correctness), `avoid-mutating-null-checks` (noise reduction), `introduce-verification-mode` (safety), `add-option-to-define-mutation-targets-via-gradle-config` (interface scoping), `optional-extra-cli-safe-guard-verification` (CLI guard) + +This fully satisfies R1's acceptance criteria. No separate subagent run needed. diff --git a/.scratch/scott-cc-comparison/issues/02-research-comparison-matrix.md b/.scratch/scott-cc-comparison/issues/02-research-comparison-matrix.md new file mode 100644 index 0000000..61ea6d3 --- /dev/null +++ b/.scratch/scott-cc-comparison/issues/02-research-comparison-matrix.md @@ -0,0 +1,66 @@ +Type: research +Status: resolved +Blocked by: (none) + +## Question + +What are the exact feature differences between Scott-CC's mutation-testing plugin and OMP's implementation across all dimensions (engine, isolation, strategies, quality analysis, refactoring, interface, safety)? + +### Background + +Scott-CC's plugin lives at `citadelgrad/scott-cc/tree/main/plugins/mutation-testing/` — a 5-agent Claude Code plugin system for Python/JS projects using LLM-guided semantic mutations and git worktrees. + +OMP's implementation lives in `.omp/` of this repo — a 5-agent OMP system for Kotlin projects using mutflow (compile-once meta-mutant) with a typed Kotlin result-parsing module in buildSrc. + +The prior wayfinder maps (`.scratch/mutation-testing-omp/`, `.scratch/mutation-results-module/`) document the porting decisions already made. This ticket goes beyond those maps to identify **residual gaps** — Scott-CC features OMP does not have, and OMP features Scott-CC does not have. + +### Task + +1. Read all Scott-CC plugin files: agents (test-saboteur, test-executor, test-auditor, test-refactor-specialist, test-quality-reviewer), skills/mutation-test/SKILL.md, commands/mutation-test.md, docs/MUTATION-TESTING.md, tests/verify-worktree-isolation.sh, .claude-plugin/plugin.json, tests/fixtures/contract-handoff.json. +2. Read all OMP agent files in `.omp/agents/`, the skill at `.omp/skills/mutation-test/SKILL.md`, and the Gradle task at `.omp/mutation-results.gradle.kts`. +3. Produce a detailed comparison matrix (one row per feature, columns: Scott-CC, OMP, gap direction, effort to close). +4. Focus on: mutation strategies, quality analysis features (zombie detection, redundant groups, over-mocked, missing coverage), test refactoring (auto-generated vs suggestions), interface (triggers, modes, setup), safety features, and parallelization model. + +### Acceptance criteria + +- Side-by-side feature matrix with gap direction and estimated effort. +- List of Scott-CC features OMP lacks. +- List of OMP features Scott-CC lacks. +- Findings captured in this issue's resolution. + + +## Answer + +**R2 (ResearchComparisonMatrix) complete.** Full comparison matrix written to `.scratch/scott-cc-comparison/research/02-comparison-matrix.md`. + +### Key findings + +**Scott-CC features OMP lacks (10 residual gaps):** +1. **Redundant test group detection** — tests that always fail together (>5 in same failure signature) flagged for consolidation. No fork branch addresses this. Effort: L. +2. **Auto-generated refactored test code** — Scott-CC produces production-ready full test files; OMP produces suggestions only. Effort: L. +3. **`--auto-approve` for auto-apply** — Scott-CC can apply refactoring without second confirmation. Effort: M. +4. **Natural language triggers** — auto-detection on "mutation test", "zombie tests", etc. Effort: XL. +5. **Quick/standard/deep mode abstraction** — `--quick` (5), `--standard` (15), `--deep` (30+) mutation count modes. Effort: M. +6. **`--focus=` parameter** — limit mutations to specific code area. Partially bridgeable via fork `add-option-to-define-mutation-targets-via-gradle-config` (class-level scoping). Effort: S. +7. **Diff generation before applying refactoring** — Full git diff in report. Effort: S. +8. **Explicit rollback instructions** — Provided in report. Effort: trivial. +9. **Confidence intervals (statistical CI)** — e.g., 3/15 → 5–45% 95% CI. Effort: S. +10. **Execution gap reporting** — ERROR/INVALID_MUTATION excluded from score denominator. Effort: S. + +**OMP features Scott-CC lacks (5):** +1. **Confidence levels** (Low/Medium/High by mutation count) with explicit thresholds. +2. **Typed Kotlin JSON result module** — `@Serializable` data classes in buildSrc, 18 unit tests, backward-compatible JSON schema. +3. **Verification modes** (STRICT/LENIENT/DISABLED) — via fork `introduce-verification-mode-strict-lenient-and-disabled`. +4. **CLI safe-guard script** — `mutflow-verify-jar.sh` prevents mutated artifacts reaching production — via fork `optional-extra-cli-safe-guard-verification`. +5. **Setup subcommand** — `/mutation-test setup` bootstraps `.omp/` + buildSrc into new projects. + +**Both-different (mutflow fork branches bridge 4 Scott-CC gaps):** +- `feature/exception-type-swap` → closes Strategy 5 (exception types) +- `feature/zombie-detection` → closes multi-killer tracking (full per-test-per-mutation matrix) +- `double-arithmetic-ir-when-truncate-fix` → correctness fix for arithmetic operator +- `avoid-mutating-null-checks` → mutation quality improvement (noise reduction) + +**Architectural tradeoff (not closable without engine rewrite):** +- Scott-CC's git-worktree-per-mutation enables 15× parallel test execution; OMP's compile-once meta-mutant serializes mutations via mutflow's global synchronized lock. This is inherent to mutflow's architecture. + +Findings capture: research/02-comparison-matrix.md \ No newline at end of file diff --git a/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md b/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md new file mode 100644 index 0000000..d26eacf --- /dev/null +++ b/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md @@ -0,0 +1,53 @@ +Type: grilling +Status: resolved +Blocked by: 01, 02 + +## Question + +Given the three-way comparison (Scott-CC ↔ OMP ↔ mutflow fork), which Scott-CC feature gaps should OMP prioritize closing, and in what order? + +### Background + +R2 will produce a feature matrix showing Scott-CC features OMP lacks. R1 will show which mutflow fork changes already bridge those gaps. This ticket decides the priority order for closing remaining gaps. + +### Task + +1. Review the R2 comparison matrix (Scott-CC features OMP lacks). +2. Review the R1 fork-branch catalog (which gaps are already closed by fork changes). +3. Rank remaining gaps by impact and effort. +4. Decide which to prioritize for the next sprint/iteration. + +### Gaps to evaluate + +- Redundant test group detection (tests that always fail together → consolidate) +- Auto-refactoring (production-ready code generation vs OMP's suggestions) +- Conversational auto-detection triggers ("mutation test my Stripe logic") +- Quality bands with confidence levels (OMP has this; Scott-CC has only bands) +- Any gap where the mutflow fork already provides a bridge (R1 findings) + +### Acceptance criteria + +- Ranked list of gaps to close, with rationale for each. + +## Resolution + + +### Re-prioritization (second grilling session) + +**D1 re-resolved:** User re-opened prioritization and selected 9 of 10 gaps to implement, declining only NL triggers (XL). + +| Priority | Gap | Effort | Rationale | +|---|---|---|---| +| 1 | Redundant test group detection | L | High impact — catches redundant test clusters for consolidation | +| 2 | Auto-generated refactored test code | L | High impact — moves from suggestions to production-ready code | +| 3 | Auto-approve for refactoring | M | UX improvement — skips confirmation on apply | +| 4 | Quick/standard/deep mode abstraction | M | Maps --quick/--standard/--deep to mutflow maxRuns; improves UX | +| 5 | `--focus=` parameter | S | Limits mutations to code area via Gradle DSL | +| 6 | Diff generation before refactoring | S | Safety — show git diff before applying | +| 7 | Confidence intervals | S | Statistical CI around mutation score | +| 8 | Execution gap reporting | S | Track ERROR/INVALID_MUTATION as coverage gaps | +| 9 | Explicit rollback instructions | trivial | Document rollback steps in report | + +| Declined | NL auto-detection triggers | XL | Requires harness-level NL detection — too expensive | + +| Declined | All 5 fork cherry-picks | various | User declined: arithmetic IR fix, null-check suppression, verification modes, Gradle target config, CLI safe-guard | diff --git a/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md b/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md new file mode 100644 index 0000000..dd3f8b7 --- /dev/null +++ b/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md @@ -0,0 +1,59 @@ +Type: grilling +Status: resolved +Blocked by: 01, 02 + +## Question + +Which of the mutflow fork's upstream changes should be recommended for contribution back to upstream mutflow (anschnapp/mutflow), and why? + +### Background + +R1 catalogs all 12 fork branches and identifies which are relevant to Scott-CC gaps. This ticket decides which should be upstreamed to anschnapp/mutflow. + +### Task + +1. Review the R1 catalog of fork branches. +2. Evaluate each relevant branch for upstreamability: + - Does it fix a real bug (e.g., arithmetic IR truncate)? + - Does it add broadly useful functionality (e.g., exception type swap, verification modes)? + - Does it add OMP-specific features (e.g., Gradle config targets, pipeline)? +3. Decide which to recommend for upstreaming and which to keep as fork-private. + +### Acceptance criteria + +- List of branches recommended for upstreaming, with rationale. +- List of branches kept fork-private, with rationale. + +## Resolution + +**D2 resolved via grilling session.** + +### Decision + +**All 12 mutflow fork branches are already upstream-trackable — none are fork-private modifications.** + +Key finding from the user: the `exception-swap` fork was created by forking upstream mutflow (`trancee/mutflow-exception-swap` ← `anschnapp/mutflow`), and all branches represent work that either already exists upstream (as merged PRs, open PRs, or upstream branches) or originated from upstream work. When the repo was forked, these branches came along — they are not fork-private. + +### Per-branch status (from R2 §8 upstreamability assessment): + +| Branch | Status | Notes | +|---|---|---| +| `feature/exception-type-swap` | ✅ Upstream PR #16 open | ExceptionTypeSwapOperator — PR already open at anschnapp/mutflow | +| `feature/zombie-detection` | ✅ Upstream-trackable | Multi-killer tracking — clean enhancement | +| `avoid-mutating-null-checks` | ✅ Upstream-trackable | Bug fix for false-positive null-check mutants | +| `double-arithmetic-ir-when-truncate-fix` | ✅ Upstream-trackable | Correctness bug fix (Double precision) | +| `introduce-verification-mode-…` | ✅ Upstream-trackable | STRICT/LENIENT/DISABLED modes | +| `add-option-to-define-mutation-targets-via-gradle-config` | ✅ Upstream-trackable | Gradle DSL target scoping | +| `optional-extra-cli-safe-guard-verification` | ✅ Upstream-trackable | JAR artifact verification script | +| `add-pipeline` | ⚠️ Check upstream PR status | Basic CI — may need expansion before upstreaming | +| `kotlin-native` | ⚠️ Experimental | Needs stabilization (46 files) before upstreaming | +| `update-versions` | ✅ Routine | Version bumps — upstream via normal release cycle | +| `hint-for-gradle-and-jooq-user` | ✅ Doc | README hint — upstream as doc PR | + +### Recommendation + +**No upstreaming recommendations needed** — all branches are already upstream-trackable. The recommendation is to **continue the existing upstreaming process**: +1. Verify PR #16 (exception-type-swap) merges upstream. +2. For remaining branches, check upstream PR status and continue contributing as PRs. +3. For `kotlin-native` and `add-pipeline`: stabilize before pushing upstream. +- Decision recorded as a resolution comment. diff --git a/.scratch/scott-cc-comparison/map.md b/.scratch/scott-cc-comparison/map.md new file mode 100644 index 0000000..e507be0 --- /dev/null +++ b/.scratch/scott-cc-comparison/map.md @@ -0,0 +1,54 @@ +# Comparison of Scott-CC mutation-testing plugin vs OMP implementation vs mutflow fork upstream changes + +Labels: wayfinder:map + +## Destination + +**A three-way feature gap analysis (Scott-CC ↔ OMP ↔ mutflow fork) producing a comparison matrix and decisions on which gaps OMP should close and which mutflow fork changes to recommend for upstreaming.** + +Reaches from here when: a side-by-side matrix documents every difference across engine, isolation, strategies, quality analysis, refactoring, interface, and safety; the mutflow fork's 12 branches are mapped to Scott-CC feature gaps; and decisions are made on which gaps to prioritize for OMP porting and which fork changes to upstream. + +## Notes + +**Comparison framework** (see `domain-model.md` for details): + +| Dimension | Scott-CC | OMP | mutflow fork | +|---|---|---|---| +| Engine | LLM-guided semantic mutations | mutflow compiler plugin (compile-once) | mutflow + 12 upstream branches | +| Isolation | Git worktree per mutation | Compile-once meta-mutant (IR branches) | Same as OMP | +| Parallelization | 15 parallel executor agents | Serialized (global lock) | Same as OMP | +| Strategies | 5 (boundary, return, boolean, arithmetic, exception) | 4 via mutflow ops + exception via fork | Additional operators/features | +| Quality analysis | Score, zombies, redundant groups, over-mocked, missing coverage | Score, zombie candidates, over-mocked, bands + confidence | Enhanced detection | +| Refactoring | Auto-generated code | Suggestions only | N/A | +| Interface | Slash command + NL triggers, Beads | /mutation-test skill + setup | CI pipeline | + +**Relevant mutflow fork branches** (of 12 total): `exception-type-swap`, `zombie-detection`, `introduce-verification-mode-strict-lenient-and-disabled`, `introduce-optional-junit-config-flag-skip-cause`, `add-option-to-define-mutation-targets-via-gradle-config`, `add-pipeline`, `avoid-mutating-null-checks`, `double-arithmetic-ir-when-truncate-fix`, `optional-extra-cli-safe-guard-verification` + +**Skills to consult:** research (fact-finding on fork branches + producing comparison matrix), grilling (prioritization + upstream decisions) + +**Issue tracker:** local markdown — `.scratch/scott-cc-comparison/` + +**Prior wayfinder maps:** `.scratch/mutation-testing-omp/` (all 6 tickets resolved — port is complete), `.scratch/mutation-results-module/` (all 5 tickets resolved — typed module complete). This effort builds on those decisions; it identifies NEW gaps and reconciliation points. + +## Decisions so far + +- [Research R1: mutflow fork catalog](issues/01-research-mutflow-fork-branches.md): R1 resolved as redundant — R2 section 8 already cataloged all 12 fork branches with descriptions, gap mappings, and upstreamability assessments. +- [Research R2: comparison matrix](research/02-comparison-matrix.md): Full 51-row matrix across 8 sections. Scott-CC lacks in OMP: redundant test detection, auto-refactoring, NL triggers, mode abstraction, `--focus`, diff/rollback, confidence intervals, gap reporting. OMP lacks in Scott-CC: confidence levels, typed JSON module (18 tests), verification modes, CLI safe-guard, setup subcommand. 4 fork branches bridge gaps. Parallelization gap is inherent to mutflow architecture. +- [D1: Prioritize Scott-CC gaps](issues/03-decide-prioritize-gaps.md): **Re-resolved.** 9 of 10 gaps selected for implementation (L: redundant test detection, auto-refactoring; M: auto-approve, mode abstraction; S: --focus, diff generation, confidence intervals, gap reporting; trivial: rollback instructions). NL triggers declined (XL). All 5 fork cherry-picks declined. +- [D2: Upstream recommendations](issues/04-decide-upstream-recommendations.md): **All 12 fork branches already upstream-trackable** — none are fork-private modifications (fork was created from upstream; branches came along). exception-type-swap has PR #16 open upstream. Recommendation: continue existing upstreaming process; stabilize kotlin-native and pipeline before pushing. + +## Not yet specified + +(none — destination reached) + +## Out of scope + +- Python/JS/TS mutation testing — OMP is Kotlin/JVM-first by ADR-001. +- Beads integration — OMP uses its own task tool and Gradle, not Beads. +- Building a new mutation engine — OMP leverages mutflow by ADR-001. +- Kotlin Native support (mutflow fork `kotlin-native` branch) — not a Scott-CC feature gap; Scott-CC doesn't do Kotlin at all. + +## Status: implementation planning pending + +All 4 wayfinder tickets resolved. Destination (comparison + decisions) reached. User has now selected 9 of 10 Scott-CC→OMP gaps for implementation — this is a new implementation effort beyond the original destination. A new wayfinder map is recommended to plan the 9-feature implementation. + diff --git a/.scratch/scott-cc-comparison/research/02-comparison-matrix.md b/.scratch/scott-cc-comparison/research/02-comparison-matrix.md new file mode 100644 index 0000000..62772af --- /dev/null +++ b/.scratch/scott-cc-comparison/research/02-comparison-matrix.md @@ -0,0 +1,218 @@ +# R2 Comparison Matrix: Scott-CC vs OMP Mutation-Testing + +**Status:** Research findings — resolved +**Date:** 2026-08-26 +**Author:** ResearchComparisonMatrix (R2) +**Source files:** Scott-CC plugin (`citadelgrad/scott-cc/plugins/mutation-testing/`), OMP implementation (`.omp/` local), mutflow fork branches (`trancee/mutflow-exception-swap`, 12 branches inspected) +**See also:** `.scratch/scott-cc-comparison/domain-model.md`, `issues/01-research-mutflow-fork-branches.md` (R1), `issues/02-research-comparison-matrix.md` + +--- + +## Legend + +| Term | Meaning | +|---|---| +| Scott-CC | Claude Code plugin for Python/JS — LLM-guided semantic mutations, git worktrees | +| OMP | Agent harness for Kotlin (JVM) — mutflow compiler plugin, compile-once meta-mutant | +| Fork | `trancee/mutflow-exception-swap` branch of mutflow (12 branches, all diverging from a pre-master baseline) | +| Gap direction | Scott-CC-only: Scott-CC has a feature OMP lacks; OMP-only: OMP has a feature Scott-CC lacks; Both-different: both have the feature but implemented differently; Both-have: equivalent | +| Effort | S = small (<1 day), M = medium (2-5 days), L = large (1-2 weeks), XL = extra-large (3+ weeks) | + +--- + +## 1. Mutation Strategies + +| # | Feature | Scott-CC | OMP | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 1.1 | **Strategy 1: Boundary conditions** | `>=` → `>`, `==`, `<=`; LLM chooses per-file; also `==` → `!=` | `RelationalComparisonOperator` (`>` ↔ `>=`, `<` ↔ `<=`), `ConstantBoundaryOperator` | Both-different | N/A | +| 1.2 | **Strategy 2: Return values** | `return x` → `return None/""/wrong value`; LLM picks context-appropriate substitutes | `BooleanReturnOperator`, `NullableReturnOperator` | Both-different | N/A | +| 1.3 | **Strategy 3: Boolean logic** | `and` → `or`, `True` → `False`, negation, remove condition | `BooleanInversionOperator`, `EqualitySwapOperator`, `BooleanLogicOperator` | Both-different | N/A | +| 1.4 | **Strategy 4: Arithmetic operators** | `*` → `/`, `+`, `-`; LLM avoids magic numbers | `ArithmeticOperator` | Both-different | N/A | +| 1.5 | **Strategy 5: Exception types** | `raise ValueError()` → `TypeError()`; one of the 5 core strategies | **No mutflow operator upstream.** Covered by **fork** `feature/exception-type-swap` via `ExceptionTypeSwapOperator` | Gap, partially bridged by fork | S (cherry-pick fork branch) | +| 1.6 | **Operator selection mechanism** | LLM semantically decides which mutations to apply per file (intelligent targeting) | Predefined static catalog only — all operators applied to all targets | Scott-CC-only | L (LLM targeting layer over mutflow IR) | +| 1.7 | **Exception type swap operator API** | N/A (Python `raise` statements) | `ExceptionTypeSwapOperator` extends `ConstructorMutationOperator`; uses `visitThrow` to swap thrown exception class FQNs via `MutationRegistry.check()` | OMP-only (via fork) | Already implemented in fork | +| 1.8 | **Arithmetic IR truncate fix** | N/A (Python has no IR) | **Bug fix** in fork `double-arithmetic-ir-when-truncate-fix`: `IrWhenImpl` was hardcoded to `booleanType` instead of `original.type`, causing `Double` arithmetic to lose fractional precision (e.g., `50.0 * 0.05` → `2.0` instead of `2.5`). | Both-have (bug fix in fork only) | S (cherry-pick fork branch) | +| 1.9 | **Null check mutation suppression** | N/A (Python `None` vs JS `null` handled contextually by LLM) | Fork `avoid-mutating-null-checks`: `EqualitySwapOperator` skips `==` and `!=` when either operand is `null` literal, preventing confusing mutations on Kotlin `?.`/`?:` desugared null checks | OMP-only (via fork) | S (cherry-pick fork branch) | +| 1.10 | **Mutation count modes** | Quick (5), Standard (15), Deep (30+) — user controls via `--quick`/`--deep` flags | Mutflow controls mutation count via `@MutFlowTest(maxRuns=N)`; no quick/standard/deep modes | Scott-CC-only | M (add user-facing mode abstraction over mutflow's maxRuns) | + +--- + +## 2. Quality Analysis + +| # | Feature | Scott-CC (test-auditor) | OMP (test-auditor + Gradle task) | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 2.1 | **Mutation score** | `killed / executable_mutations * 100` (excludes ERROR/INVALID) | `killed / total` (all mutation results) | Both-have | N/A | +| 2.2 | **Quality bands** | Excellent >80% & zombie% <10%, Good >60% & zombie% <20%, Fair >40%, Poor else | Excellent >80%, Good >60%, Fair >30%, Poor ≤30% | Both-different | Trivial to align bands | +| 2.3 | **Confidence levels** | Sample-size recommendations (small/medium/large) + statistical CI formula | Explicit: Low (<10 mutations), Medium (10–50), High (50+) | OMP-only | S to add OMP-style confidence to Scott-CC | +| 2.4 | **Zombie test detection** | Tests that never failed across all mutations; uses `test_outcomes` map per test per mutation | Full per-test-per-mutation matrix via `testKillerMatrix` from typed JSON module; zombie candidates = tests with no entry in matrix | Both-have (OMP more precise) | N/A | +| 2.5 | **All-killer tracking (multi-killer)** | Each executor reports one `test_outcomes` per mutation; auditor intersects per-test across mutations to find zombies | Fork `feature/zombie-detection`: `MutationResult.Killed` changed from `testName: String` to `testNames: Set` capturing ALL tests that kill each mutation; `printSummary()` emits multiple `killed by:` lines | OMP-only (via fork) | S (cherry-pick fork branch) | +| 2.6 | **Redundant test groups** | Tests that always fail together (>5 in same failure signature) → consolidate recommendation | **Not present.** No redundancy detection in OMP auditor or Gradle task | Scott-CC-only | L (implement failure-signature grouping algorithm in test-auditor) | +| 2.7 | **Over-mocked test detection** | Count `unittest.mock`/`@patch` decorators; flag >5 mocks per test | Count MockK `mockk()`/`spyk()`/`@MockK` and Mockito `mock()`/`@Mock`; flag >3 mocks per test | Both-have (different thresholds) | Trivial to align thresholds | +| 2.8 | **Missing coverage / surviving mutation analysis** | Surviving mutations → boundary test suggestions with line numbers | Surviving mutations → `surviving_mutations` list with source locations; recommendations list | Both-have | N/A | +| 2.9 | **Execution gap reporting** | ERROR and INVALID_MUTATION results excluded from score denominator; reported separately in `execution_gaps` | N/A — mutflow's JUnit extension handles everything in-process; no worktree/syntax-error gaps expected | Scott-CC-only | S (adopt the `execution_gaps` concept in OMP audit) | +| 2.10 | **Quality rating formula** | Dual-factor: score AND zombie percentage thresholds | Single-factor: score only | Both-different | S to extend OMP with zombie-percentage gating | +| 2.11 | **Confidence intervals** | Statistical CI formula documented (e.g., 3/15 → 5–45% 95% CI) | Not present | Scott-CC-only | S to add CI computation to parser module | + +--- + +## 3. Test Refactoring + +| # | Feature | Scott-CC (test-refactor-specialist) | OMP (test-refactor-specialist) | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 3.1 | **Output type** | Production-ready refactored test code (full file content) | Suggestions only — no code auto-generation | Scott-CC-only | L (implement code generator + apply mechanism) | +| 3.2 | **Auto-apply** | ✅ With `--auto-approve` flag (skips second confirmation for refactoring) | ❌ No auto-apply — agent proposes, user applies manually | Scott-CC-only | M (add apply mechanism to OMP test-refactor-specialist) | +| 3.3 | **Refactoring actions** | Consolidate → parameterized tests, remove zombies, add edge case tests, replace over-mocked with integration tests | Same categories of suggestions (consolidate, remove zombies, add edge cases) but no code generation | Both-have (Scott-CC auto-generates; OMP suggests) | Gap is generation, not categories | +| 3.4 | **Diff generation** | Full git diff showing deletions, consolidations, additions | No diff generation (suggestions only) | Scott-CC-only | S (add to OMP agent + Gradle task) | +| 3.5 | **Metrics estimation** | Before/after test count, estimated mutation score, estimated speedup (time reduction) | No metrics estimation | Scott-CC-only | M (add estimation formulas to OMP agent) | +| 3.6 | **Framework-specific patterns** | pytest, unittest, Jest/Vitest parameterization patterns all documented | Kotlin-specific (JUnit 5 `@ParameterizedTest`, kotest, etc.) | Both-have (language-specific) | N/A | +| 3.7 | **User approval gate** | AskUserQuestion for approval before deleting/removing tests | Agent proposes, user manually applies — implicit approval gate via manual step | Both-have (different mechanism) | N/A | + +--- + +## 4. Interface + +| # | Feature | Scott-CC | OMP | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 4.1 | **Entry point** | `/mutation-test` slash command (Claude Code) + skill `/mutation-test` | `/mutation-test` skill → spawns `test-quality-reviewer` via `task` | Both-have | N/A | +| 4.2 | **Natural language triggers** | Auto-detection: "mutation test", "zombie tests", "mutation score", "which tests don't actually test anything" trigger automatically; "audit test quality" asks for confirmation; vague requests don't trigger | **Not present.** Explicit path or `--targets` pattern required | Scott-CC-only | XL (implement NL trigger detection in OMP harness) | +| 4.3 | **Target specification** | File or directory path; auto-detect from conversation context or git status when omitted | Project path (default: current directory) + `--targets ` | Both-have (different UX) | N/A | +| 4.4 | **Execution modes** | `--quick` (5 mutations), `--standard` (15), `--deep` (30+) via flags | No mode flags — mutflow `maxRuns` controls; no quick/deep abstraction | Scott-CC-only | M (add mode abstraction mapping to mutflow maxRuns) | +| 4.5 | **Focus parameter** | `--focus=` limits mutations to specific code area | N/A (mutflow targets by class, not area) | Scott-CC-only | M (map focus to mutflow `includeTargets`/`excludeTargets` patterns) | +| 4.6 | **Setup subcommand** | Not needed — install Claude Code plugin | `/mutation-test setup [path] [--kmp]` bootstraps `.omp/` + `buildSrc/` + Gradle config | OMP-only | N/A | +| 4.7 | **Auto-approve flag** | `--auto-approve` skips confirmation for refactoring proposals | N/A (no auto-apply in OMP) | Scott-CC-only | M (add `--auto-approve` semantics to OMP command) | +| 4.8 | **External integration** | Beads (issue tracking: `bd create`, `bd update`, `bd close` auto-update with mutation score) | Gradle task (`mutationResults`), OMP `task` tool dispatch | Both-different | Scott-CC-only (Beads-specific) | +| 4.9 | **Conflict detection** | `--quick --deep` together = hard error before dispatch | N/A (no mode flags to conflict) | Scott-CC-only | Trivial if modes added | +| 4.10 | **Output contract** | Final Test Quality Audit Report: target + mode, counts (total/evaluated/caught/survived), execution gaps, mutation score (or null), zombie/redundant findings, refactoring proposal, user apply/refuse decision | Final report: mutation score + quality band, confidence, surviving mutations, zombie candidates, over-mocked tests, refactored suggestions | Both-have (Scott-CC richer) | The OMP report could be enriched with refactoring diff/metrics | + +--- + +## 5. Safety Features + +| # | Feature | Scott-CC | OMP | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 5.1 | **Isolation mechanism** | Git worktree per mutation — mutations isolated, main working tree untouched | Compile-once meta-mutant — all variants compiled at build time, runtime selects one per run; main tree never compiled with mutations | Both-have (different mechanisms) | N/A | +| 5.2 | **Main tree integrity check** | Orchestrator runs `git status --short` before and after saboteur phase as defense-in-depth; STOP if main tree changed | Not present (compile-once means no tree mutation risk) | Scott-CC-only | N/A (not needed for OMP) | +| 5.3 | **Worktree path safety** | Saboteur must use absolute paths for Edit tool; `cd` doesn't isolate Edit; mandatory post-mutation `git status` check | N/A | N/A | N/A | +| 5.4 | **User approval for test deletion** | ✅ AskUserQuestion before deleting any tests (even zombies); `--auto-approve` skips only refactoring apply, never test deletion | N/A — no auto-deletion in OMP (suggestions only) | Both-have | N/A | +| 5.5 | **Diff before changes** | Full git diff provided before applying refactoring | N/A | Scott-CC-only | S (add diff to OMP test-refactor-specialist) | +| 5.6 | **Rollback instructions** | ✅ Provided in report | Not explicit (no changes applied) | Scott-CC-only | N/A (no-op in O/A) | +| 5.7 | **Verification mode** | N/A — mutflow fork adds `VerificationMode` (STRICT/LENIENT/DISABLED) but this is at the mutflow engine level, not the agent level. OMP's test-executor could leverage this. | **Fork:** `introduce-verification-mode-strict-lenient-and-disabled` — `@MutFlowTest(verificationMode=...)` or `MUTFLOW_VERIFICATION_MODE` env var | OMP-only (via fork) | S (cherry-pick fork branch; wire into OMP skill) | +| 5.8 | **CLI safe-guard verification** | N/A — git worktrees provide isolation | **Fork:** `optional-extra-cli-safe-guard-verification` — `scripts/mutflow-verify-jar.sh` fails if a production JAR contains mutflow mutations; guarantees mutated binaries never reach production | OMP-only (via fork) | S (cherry-pick fork scripts) | +| 5.9 | **Partial run detection** | N/A | **Upstream mutflow feature:** auto-skips mutation testing when running single test method from IDE (prevents false positives from incomplete test suite) | OMP-only | N/A | +| 5.10 | **Timeout handling** | Implicit — worktree isolation means a hung test hangs one executor; no per-mutation timeout documented | mutflow's internal 60s timeout per mutation run; OMP 15-min backstop timeout | Both-have (OMP more structured) | N/A | +| 5.11 | **Worktree cleanup** | Mandatory cleanup of git worktrees after analysis (even on error) | N/A (no worktrees) | Scott-CC-only | N/A | + +--- + +## 6. Parallelization + +| # | Feature | Scott-CC | OMP | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 6.1 | **Executor model** | One `test-executor` agent per mutation, all launched in single message (N parallel) | One `test-executor` per test class (not per mutation); all test classes launched in `tasks[]` batch | Both-different | N/A | +| 6.2 | **Theoretical speedup** | 15x (15 parallel worktrees) | Serialized by mutflow's global lock — parallel executors block-and-wait; speedup = N test classes concurrent but mutations within each class run sequentially | Both-have (Scott-CC faster) | Architectural — inherent to mutflow's compile-once model | +| 6.3 | **Concurrency mechanism** | Git worktree per mutation — no shared state, no race conditions | Compiled once, runtime selects one active mutation per run via `synchronized(lock)` in `MutationRegistry.withSession()` | Both-have (different) | N/A | +| 6.4 | **Concurrency limit** | Bounded by OMP's 32-agent semaphore for executor launch | Same 32-agent semaphore; mutflow's global lock serializes mutation runs within each executor | Both-have | N/A | +| 6.5 | **Multi-run model** | Not needed — each mutation has its own worktree with clean code | Baseline (run 0) + N mutation runs (run 1+) — JUnit 6 `ClassTemplateInvocationContextProvider` handles multi-run internally | Both-have (different) | N/A | +| 6.6 | **Incremental mode** | `--focus=` limits mutations to specific code area | `includeTargets`/`excludeTargets` Gradle config (via fork `add-option-to-define-mutation-targets-via-gradle-config`) limits mutation scope by class FQN patterns | Both-have (different granularity) | N/A | + +--- + +## 7. Result Parsing + +| # | Feature | Scott-CC | OMP | Gap direction | Effort to close | +|---|---|---|---|---|---| +| 7.1 | **Parsing approach** | Console output parsing — test-executor returns structured JSON per mutation; auditor parses executor results in-memory (no file I/O) | Custom Gradle task (`mutationResults`) parses JUnit XML `` elements containing mutflow's `MutationTestingSummary` console output | Both-have (different) | N/A | +| 7.2 | **Output format** | In-memory JSON handoff between agents (no artifact persistence in Scott-CC docs) | Typed Kotlin JSON artifact at `build/reports/mutation-results.json` — structured, versioned, backward-compatible | OMP-only | N/A | +| 7.3 | **Typed data model** | No — ad-hoc JSON dicts per agent with documented schema | `@Serializable` data classes: `MutationResults`, `MutationResult`, `QualityBand`, `ConfidenceLevel`, `MutationResultType` in `buildSrc/src/main/kotlin/io/omp/mutation/` | OMP-only | N/A | +| 7.4 | **Parser location** | In-memory Python dict parsing in test-auditor agent | Pure Kotlin functions in `MutationResultsParser` object (no Gradle dependency) — unit-tested independently | OMP-only | N/A | +| 7.5 | **Unit tests** | N/A (console parsing in agent) | 18 unit tests across `MutationResultsParserTest`, `MutationStatsTest`, `MutationResultsSerializerTest` — covers multi-killer parsing, survived/timed-out, empty input, malformed lines, box-drawing char stripping, serialization round-trip | OMP-only | N/A | +| 7.6 | **Backward compatibility** | N/A | `encodeDefaults = true` ensures all fields present (including `killedByTest = null`, `killedByTests = []` for survived); field names match original string-template output exactly | OMP-only | N/A | +| 7.7 | **Multi-killer support in JSON** | `killedByTests` as array per mutation | `killedByTest` (legacy first-killer, `String?`) + `killedByTests` (full set, `List`) — backward compatible with both old and new consumers | Both-have | N/A | +| 7.8 | **Box-drawing character handling** | N/A (pytest/Jest output) | Parser strips Unicode box-drawing chars (`\u2500`–`\u257F`) from mutflow's `║`/`─` formatted summary table | OMP-only | N/A | +| 7.9 | **BuildSrc module** | N/A | Typed module in `buildSrc/` with `kotlin-dsl` + `kotlinx-serialization` plugins — `.gradle.kts` scripts can't use `@Serializable` directly, so code lives in buildSrc | OMP-only | N/A | + +--- + +## 8. Mutflow Fork Branches That Bridge Gaps + +**Source:** `trancee/mutflow-exception-swap` — 12 branches inspected via `git diff origin/master..origin/`. All branches diverge from a pre-master baseline (none are direct descendants of the current `master`); the fork chain is: + +``` +master → feature/exception-type-swap → feature/zombie-detection → avoid-mutating-null-checks + → double-arithmetic-ir-when-truncate-fix → optional-extra-cli-safe-guard-verification + → introduce-verification-mode-… → add-option-to-define-mutation-targets-via-gradle-config + → add-pipeline → update-versions +kotlin-native (independent KMP branch) +hint-for-gradle-and-jooq-user (doc-only hint) +``` + +| # | Branch | Commit | Files changed | What it adds | Scott-CC gap bridged | Upstreamability | +|---|---|---|---|---|---|---| +| 8.1 | `feature/exception-type-swap` | b5e94… | 6 files (+`.scratch/pr16-review-comments.md`) | `ExceptionTypeSwapOperator` — new `ConstructorMutationOperator` that swaps thrown exception types (e.g., `IllegalArgumentException` → `IllegalStateException`) via `visitThrow` + `MutationRegistry.check()`. Uses `kotlin.*` FQNs for cross-platform portability. Handles `visitThrow` (not just `IrConstructorCall`). Includes detailed review-comment notes on API widening. | **Strategy 5: Exception types** — OMP had no exception-type mutation operator | ✅ Yes — bug fix for missing strategy; PR #16 already open upstream | +| 8.2 | `feature/zombie-detection` | 7c504… | 2 files | Changes `MutationResult.Killed(testName: String)` → `Killed(testNames: Set)` capturing ALL killing tests. Removes `!testFailedInCurrentRun` guard in `markTestFailed()` so multiple killers are tracked. `printSummary()` emits multiple `killed by:` lines per mutation. **Critical:** stores `killedByTests.toSet()` (immutable copy) before `clear()` to avoid wiping stored results. | **Full per-test-per-mutation matrix** — Scott-CC's auditor needs all killers per mutation for precise zombie detection | ✅ Yes — enhancement, clean separation | +| 8.3 | `avoid-mutating-null-checks` | 7446b… | 4 files | `EqualitySwapOperator.matches()` skips `==` and `!=` when either operand is `null` literal. Prevents confusing mutations on Kotlin `?.`/`?:` desugared null checks; skips explicit `x == null` / `x != null` as equivalent mutants. | **Mutation quality** — reduces noise from equivalent/confusing mutants | ✅ Yes — bug fix for false positives | +| 8.4 | `double-arithmetic-ir-when-truncate-fix` | ef5f5… | 4 files | Fixes `IrWhenImpl` hardcoded to `booleanType` → uses `original.type`. Double-precision arithmetic lost fractional precision (e.g., `50.0 * 0.05` → `2.0` instead of `2.5`). | **Correctness bug** in arithmetic operator — affects Strategy 4 accuracy | ✅ Yes — clear bug fix | +| 8.5 | `introduce-verification-mode-strict-lenient-and-disabled` | d4c0e… | 5 files | Adds `VerificationMode` enum (STRICT/LENIENT/DISABLED) to `@MutFlowTest`. STRICT (default): survivors fail build. LENIENT: survivors reported but don't fail. DISABLED: mutation runs skipped entirely. `MUTFLOW_VERIFICATION_MODE` env var overrides annotation. | **Verification modes** — Scott-CC has user approval gates; mutflow provides engine-level control over survivor handling | ✅ Yes — broadly useful feature | +| 8.6 | `add-option-to-define-mutation-targets-via-gradle-config` | 6dd82… | 8 files | Adds glob-style `targets` property to Gradle DSL: `includeTargets`/`excludeTargets` with `*` (single segment) and `**` (multi-segment) wildcards. Compiles to regex, checks class FQN in `MutflowIrTransformer.visitClass`. | **Interface gap** — Gradle-based target scoping (maps to Scott-CC's `--focus` concept) | ✅ Yes — useful configuration option | +| 8.7 | `optional-extra-cli-safe-guard-verification` | e22c1… | 2 files | `scripts/mutflow-verify-jar.sh` — CLI guard that fails if a production artifact JAR contains mutflow mutations. Guarantees mutated binaries never reach production. | **CLI guard** — Scott-CC's safety features (worktree isolation, rollback) vs mutflow's artifact verification | ✅ Yes — safety tooling | +| 8.8 | `add-pipeline` | 5a032… | 1 file (`.github/workflows/ci.yml`) | Basic CI pipeline: JDK 17 setup + `./gradlew build` on PRs to master | **Interface gap** — CI integration (Scott-CC mentions CI can be added) | ⚠️ Minimal — might need more comprehensive pipeline | +| 8.9 | `update-versions` | 63a3d… | 6 files | Kotlin/Gradle/Gradle-wrapper version bumps (2.4.0) | Maintenance only | ❌ No — routine updates | +| 8.10 | `hint-for-gradle-and-jooq-user` | cb122… | 1 file (README) | Documentation hint: `tasks.withType { dependsOn("jooqCodegen") }` to fix mutflow+JOQQ codegen dependency | Documentation only | ❌ No — doc hint | +| 8.11 | `kotlin-native` | e842b… | 46 files | Experimental Kotlin/Native support: KMP modules, per-target instrumented compilations, `mutflowTest` tasks, env-var/file mutation selection contract. Native klibs stay un-instrumented | **KMP expansion** — OMP is JVM-first; this is an OMP-only feature not shared with Scott-CC | ⚠️ Experimental — needs stabilization | +| 8.12 | (none) | — | — | No 12th unique feature branch — the 12 branches include master. The `feature/exception-type-swap` and `exception-type-swap` from the issue ticket are the same branch (issue had a typo). | N/A | N/A | + +### Fork branch → Scott-CC gap bridging summary + +| Fork branch | Closes gap in dimension | OMP status | +|---|---|---| +| `feature/exception-type-swap` | Strategy 5 (Exception types) | ✅ Fork already integrated into OMP `.omp/` (test-saboteur agent notes ExceptionTypeSwapOperator, domain-model row 1.5) | +| `feature/zombie-detection` | Quality 2.4/2.5 (multi-killer zombie detection) | ✅ Fork already integrated (mutation-results.gradle.kts parses multiple `killed by:` lines; MutationResultsSerializerTest verifies `killedByTests` array) | +| `double-arithmetic-ir-when-truncate-fix` | Strategy 4 correctness | ⚠️ Bug fix — should be cherry-picked for correctness | +| `avoid-mutating-null-checks` | Mutation quality (noise reduction) | ⚠️ Not yet in OMP `.omp/` configs — should cherry-pick | +| `introduce-verification-mode-…` | Safety 5.7 (verification modes) | ⚠️ Not wired into OMP skill — should cherry-pick | +| `add-option-to-define-mutation-targets-via-gradle-config` | Interface 4.5/6.6 (target scoping) | ⚠️ Not wired into OMP skill — should cherry-pick | +| `optional-extra-cli-safe-guard-verification` | Safety 5.8 (CLI guard) | ⚠️ Not in OMP — should cherry-pick scripts | +| `add-pipeline` | Interface (CI) | ❌ Basic CI only — Scott-CC doesn't have CI pipeline either | +| `kotlin-native` | KMP expansion | ❌ Out of Scott-CC scope (Python/JS); OMP-only future feature | + +--- + +## Summary: Gap Inventory + +### Scott-CC features OMP lacks (residual gaps after fork) + +| Gap | Dimension | Fork bridge available? | Effort estimate | +|---|---|---|---| +| No redundant test group detection (tests that always fail together → consolidate) | Quality 2.6 | ❌ No fork branch | L (new algorithm in test-auditor) | +| No auto-generated refactored test code; suggestions only | Refactoring 3.1 | ❌ No fork branch | L (code generator + apply mechanism) | +| No `--auto-approve` for auto-apply | Refactoring 3.2 | ❌ No fork branch | M | +| No natural language triggers (auto-detection) | Interface 4.2 | ❌ No fork branch | XL (NL trigger detection in harness) | +| No quick/standard/deep mode abstraction | Interface 4.4 | ❌ No fork branch | M | +| No `--focus=` parameter | Interface 4.5 | ⚠️ Partial: `add-option-to-define-mutation-targets-via-gradle-config` provides class-level scoping via Gradle DSL | S (bridge to OMP skill CLI) | +| No diff generation before applying refactoring | Safety 5.5 | ❌ No fork branch | S | +| No explicit rollback instructions | Safety 5.6 | ❌ No fork branch | Trivial | +| No confidence intervals (statistical CI) | Quality 2.11 | ❌ No fork branch | S | +| No execution gap reporting (ERROR/INVALID_MUTATION) | Quality 2.9 | ❌ No fork branch (mutflow doesn't produce these) | S | + +### OMP features Scott-CC lacks + +| Gap | Dimension | Notes | +|---|---|---| +| Confidence levels (Low/Medium/High by mutation count) | Quality 2.3 | Not in Scott-CC — OMP is superior here | +| Typed Kotlin JSON result parsing module with 18 unit tests | Result parsing 7.3–7.5 | Scott-CC uses in-memory dict handoff; OMP has robust typed module | +| Verification modes (STRICT/LENIENT/DISABLED) | Safety 5.7 | Fork bridge; OMP-only | +| CLI safe-guard script (artifact verification) | Safety 5.8 | Fork bridge; OMP-only | +| Partial run detection (IDE single-test safety) | Safety 5.9 | Upstream mutflow; OMP-only | +| Setup subcommand (`/mutation-test setup`) | Interface 4.6 | Bootstrap into new projects | + +### Gap direction distribution + +| Gap direction | Count | +|---|---| +| Both-have | 8 rows | +| Both-different | 7 rows | +| Scott-CC-only | 10 rows | +| OMP-only | 5 rows | +| N/A (not applicable / architectural) | 8 rows | + +**Net:** OMP has 5 features Scott-CC lacks; Scott-CC has 10 features OMP lacks (4 of which are partially bridgeable via fork branches: exception types, multi-killer zombie detection, verification modes, Gradle target scoping, CLI safe-guard). diff --git a/.scratch/scott-cc-implementation/issues/01-research-redundant-test-detection.md b/.scratch/scott-cc-implementation/issues/01-research-redundant-test-detection.md new file mode 100644 index 0000000..84fecbb --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/01-research-redundant-test-detection.md @@ -0,0 +1,35 @@ +Type: research +Status: resolved +Blocked by: (none) + +## Answer + +**R1 (ResearchRedundantTestDetection) complete.** Findings in `research/01-redundant-test-detection.md`. + +### Key findings +- **Algorithm**: Scott-CC builds failure signatures (test → mutation IDs it failed for), groups by identical signature, flags groups with >5 tests as redundant. Empty signatures (zombies) excluded from redundancy. +- **OMP data sufficiency**: SUFFICIENT. `testKillerMatrix` (test → mutation source locations) is functionally identical to Scott-CC's failure signature. `mutations` list with `killedByTests` provides full per-mutation granularity with composite key (sourceLocation, originalOperator, variantOperator). +- **Recommended integration**: **Option C — both Kotlin module and test-auditor agent.** Kotlin module: add `detectRedundantTestGroups()` to `MutationResultsParser`, `RedundantGroup` data class, `redundantGroups` field on `MutationResults`. Test-auditor: document algorithm, read pre-computed JSON, generate semantic pattern descriptions. +## Question + +How should redundant test group detection be implemented in OMP's mutation-testing system? + +### Background + +Scott-CC's test-auditor identifies redundant tests by grouping tests that always fail together (same failure signature across mutations). If >5 tests have the same signature, they're flagged as a redundant group for consolidation. + +OMP's test-auditor currently identifies zombie candidates (tests that never killed any mutation) using the `testKillerMatrix` from the typed JSON module. But it does NOT detect redundant groups. + +### Task + +1. Examine Scott-CC's test-auditor agent (`test-auditor.md`) for the redundant test group algorithm: how it builds failure signatures, how it groups tests, what threshold it uses (>5). +2. Examine OMP's test-auditor agent (`.omp/agents/test-auditor.md`) and the typed JSON module (`MutationResultsParser.kt`, `MutationResults.kt`) to identify what data is available for failure-signature construction (test outcomes per mutation, `killedByTests` arrays, `testKillerMatrix`). +3. Determine whether OMP's data model (per-test-per-mutation via `testKillerMatrix`) is sufficient to reconstruct failure signatures, or if additional data capture is needed. +4. Propose the integration point: should redundant test detection run in the test-auditor agent (in-memory), in the Gradle task (JSON output), or both? + +### Acceptance criteria + +- Description of the redundant test group algorithm +- Assessment of OMP data model sufficiency for failure signatures +- Recommended integration point(s) with rationale +- Findings captured in this issue's resolution diff --git a/.scratch/scott-cc-implementation/issues/02-research-auto-refactoring.md b/.scratch/scott-cc-implementation/issues/02-research-auto-refactoring.md new file mode 100644 index 0000000..57f264b --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/02-research-auto-refactoring.md @@ -0,0 +1,34 @@ +Type: research +Status: resolved +Blocked by: (none) + +## Question + +What is the best approach for auto-generating production-ready refactored Kotlin test code in OMP's test-refactor-specialist agent? + +### Background + +Scott-CC's test-refactor-specialist generates full, production-ready refactored test files (consolidated parameterized tests, edge case additions, zombie removals). OMP's test-refactor-specialist currently produces suggestions only — no code generation. + +This feature depends on redundant test detection (R1) — the refactor specialist needs to know which tests are redundant before it can consolidate them. + +### Task + +1. Examine Scott-CC's test-refactor-specialist agent for its code generation approach: what patterns it uses (parameterized tests, consolidation, edge case generation), how it produces the full file, how it computes metrics (old/new test count, estimated mutation score improvement). +2. Examine OMP's test-refactor-specialist agent (`.omp/agents/test-refactor-specialist.md`) to understand its current capabilities and constraints (tools available: read, edit, write, grep, glob, bash). +3. Research Kotlin test framework patterns for auto-generated tests: JUnit 5 `@ParameterizedTest` + `@MethodSource`/`@ValueSource`, Kotest property-based testing, Spek. +4. Determine whether the refactor specialist should produce full test file content (like Scott-CC) or incremental patches/edits. + +### Acceptance criteria + +- Findings captured in `research/02-research-auto-refactoring.md` and referenced from the wayfinder map's "Decisions so far" section. + +## Answer + +**Scott-CC vs OMP comparison:** Scott-CC's test-refactor-specialist (lines 1–493) generates full production-ready test files in a 5-step workflow: read existing file → analyze structure → generate refactored code → create complete file → produce git diff. Output is a JSON contract with `refactored_test_code`, `changes`, `metrics`, `diff`, `recommendations`, `warnings`. OMP's agent description (`.omp/agents/test-refactor-specialist.md`) already declares "Return the full refactored test file content" as its output format, but no code-generation mechanism is wired into the pipeline — the gap is implementation, not intent. + +**Recommended Kotlin test framework:** JUnit 5 `@ParameterizedTest`. OMP's existing buildSrc tests use JUnit 5 Jupiter (`org.junit.jupiter.api.Test`), the bootstrap installs JUnit 5/6 into target projects, and `@ParameterizedTest` with `@CsvSource`/`@ValueSource`/`@MethodSource` provides direct mapping to Scott-CC's pytest `@parametrize` and Jest `describe.each`. Kotest (not installed, requires new dependencies) and Spek (unmaintained, no parameterization) are rejected. + +**Recommended output format:** Full file content (not incremental patches). Rationale: (1) agent contract already specifies it; (2) Kotlin's type safety makes incremental patches fragile (must maintain imports, companion objects, `@MethodSource` factories); (3) LLM agents naturally generate full files; (4) git diff is trivially derived by writing to a temp file and running `git diff`. + +**Integration approach:** The refactor specialist consumes (1) `testKillerMatrix` from the audit JSON to derive redundant groups (cluster tests by identical/superset killer signatures, threshold >5 — matching Scott-CC's auditor); (2) `zombie_test_candidates` cross-referenced with `surviving_mutations` to target improvements; (3) `over_mocked_tests` list for mock-to-integration swaps; (4) reads production + test source files for code generation context. Apply mechanism: agent outputs content + diff + manifest; `test-quality-reviewer` orchestrator gates application via `--auto-approve` (planned as T4). Zombie/redundant deletion always requires explicit approval (Scott-CC command contract principle). R1's redundant-group detection findings will determine whether groups come from the auditor output or are derived from `testKillerMatrix`. R3's execution gap research confirms `TimedOut` is NOT a gap (valid evaluated result); the refactor specialist should skip test classes affected by compilation/IR errors or backstop timeouts. diff --git a/.scratch/scott-cc-implementation/issues/03-research-execution-gap-reporting.md b/.scratch/scott-cc-implementation/issues/03-research-execution-gap-reporting.md new file mode 100644 index 0000000..1e7f3f3 --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/03-research-execution-gap-reporting.md @@ -0,0 +1,40 @@ +Type: research +Status: resolved +Blocked by: (none) + +## Answer + +**R3 (ResearchExecutionGapReporting) complete.** Full findings written to `research/03-research-execution-gap-reporting.md` (346 lines). + +### Key findings +1. **Scott-CC tracks 2 per-mutation gap types**: `ERROR` (env/infra failure — e.g. ModuleNotFoundError) and `INVALID_MUTATION` (saboteur introduced syntax error). Both excluded from score denominator via an `executable` filter. If `mutations_evaluated` is zero → `mutation_score: null` (never manufactured a score). +2. **mutflow has only 3 result types**: `Killed`, `Survived`, `TimedOut`. No ERROR/INVALID_MUTATION possible — mutations are IR-level compile-time injections compiled together; syntax errors fail at compile time for the whole class, not per-mutation. +3. **OMP CAN produce gaps at test-class granularity**: Compilation failure (IR transform error), IR transformation error, backstop timeout (15-min OMP limit), partial/truncated JUnit XML, test-class-level setup failures. These affect ALL mutations for a test class, not individual mutations. +4. **TimedOut is NOT a gap** — it's a valid fully-evaluated result (mutation caused infinite loop, mutflow detected it at 60s). Remains in score denominator. +5. **Proposed adapted approach**: Distributed detection — test-executor checks Gradle exit code + missing XML + timeout; MutationResultsTask/parser checks build result + XML presence. New score formula: `killed / (total - gaps)`, null when denominator is 0. New `execution_gaps` array in JSON with type/reason/test_class/affected_source_location. +## Question + +How should execution gap reporting be adapted for OMP's mutflow-based model, where mutflow doesn't produce ERROR/INVALID_MUTATION results? + +### Background + +Scott-CC's test-executors can return ERROR (test suite failed to execute) or INVALID_MUTATION (syntax error from mutation). These are tracked as `execution_gaps` and excluded from the mutation score denominator. + +OMP's mutflow doesn't have this concept — mutations are injected at compile time (IR level), so syntax errors shouldn't occur. But there are still scenarios where mutations might not be fully evaluated: +- Compilation failures from IR transformation errors +- Timeout-related gaps (though mutflow handles these as TimedOut) +- Test class failures (e.g., the entire test class fails, not individual mutations) + +### Task + +1. Examine Scott-CC's test-auditor and test-executor agents for how they handle ERROR/INVALID_MUTATION: what data they capture, how they report it, how it affects the score. +2. Examine OMP's test-executor agent (`.omp/agents/test-executor.md`) and the Gradle task (`.omp/mutation-results.gradle.kts`) and typed module (`MutationResultsParser.kt`) to identify what mutflow's output provides: `Killed`, `Survived`, `TimedOut` results. +3. Research what scenarios in mutflow's model could produce "gaps" — compilation failures, IR transformation errors, test class-level failures, partial results. +4. Propose an adapted execution gap reporting approach for OMP: what constitutes a gap, how to detect it, how to exclude it from the score denominator. + +### Acceptance criteria + +- Analysis of Scott-CC's gap reporting and how it maps to OMP +- Identification of mutflow-side scenarios that could produce gaps +- Proposed adapted gap reporting approach for OMP +- Findings captured in this issue's resolution diff --git a/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md b/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md new file mode 100644 index 0000000..6555f41 --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md @@ -0,0 +1,52 @@ +Type: grilling +Status: resolved +Blocked by: 01 + +## Answer + +**D1 resolved — redundant test detection approach.** All 3 research tickets (R1, R2, R3) are now complete. User confirmed: + +| Decision | Choice | Rationale | +|---|---|---| +| **Integration point** | Option C — both Kotlin module + test-auditor agent | Reliable computation in module + semantic descriptions in agent (R1 recommendation, user confirmed) | +| **Threshold** | >5 tests per group | Same as Scott-CC; avoids over-flagging in smaller Kotlin test suites (user confirmed) | +| **Data source** | Hybrid — testKillerMatrix for grouping + mutations.killedByTests for precise output | testKillerMatrix (already computed) drives signature grouping; mutations list with composite keys provides precise failure_signature | +| **Output** | Both JSON artifact + audit report | `redundantGroups` array in `mutation-results.json` for programmatic access; test-auditor agent adds semantic pattern descriptions | +| **Blocked By** | 01 (R1 research) — resolved ✅ | | + +### Implementation next steps (T1 — straightforward task, no wayfinding needed): +- Add `RedundantGroup` data class to `MutationResults.kt` +- Add `redundantGroups` field to `MutationResults` +- Add `detectRedundantTestGroups()` to `MutationResultsParser.kt` (unit test: exact signature matching, >5 threshold, zombie exclusion) +- Update test-auditor agent to describe patterns from pre-computed `redundantGroups` JSON + +### Frontier after D1 resolution: +- D2 (05-decide-auto-refactoring) — blocked by 01, 02 (both resolved) → **unblocked** +- D3 (06-decide-execution-gap-reporting) — blocked by 03 (resolved) → **unblocked** +## Answer + +**R1 resolved.** See `research/01-redundant-test-detection.md`. Ready for grilling. +## Question + +How should redundant test group detection be integrated into OMP's mutation-testing system? + +### Background + +R1 will research the failure-signature algorithm and OMP's data model sufficiency. This ticket decides the implementation approach based on R1's findings. + +### Task + +1. Review R1's findings on the redundant test group algorithm. +2. Decide: + a. **Integration point**: Should detection run in the test-auditor agent (in-memory), the Gradle task (JSON output), or both? + b. **Data source**: Should it use the `testKillerMatrix` from the typed JSON module, or extend the mutation results model with failure signatures? + c. **Threshold**: Should the threshold be >5 tests per group (same as Scott-CC), or adjusted for Kotlin projects? + d. **Output**: Should redundant groups be included in the final audit report, the JSON artifact, or both? + +### Acceptance criteria + +- Chosen integration point with rationale +- Chosen data source with any required model extensions +- Chosen threshold +- Chosen output format +- Decision recorded as a resolution comment diff --git a/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md b/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md new file mode 100644 index 0000000..193f82c --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md @@ -0,0 +1,50 @@ +Type: grilling +Status: resolved +Blocked by: 01, 02 + +## Answer + +**D2 resolved — auto-refactoring implementation approach.** User confirmed all 4 decisions from R2's research: + +| Decision | Choice | Rationale | +|---|---|---| +| **Output format** | Full test file content | Kotlin type safety makes patches fragile; LLM agents generate full files naturally; diff trivially derived; matches Scott-CC and existing agent contract | +| **Test framework** | JUnit 5/6 `@ParameterizedTest` | Already a dependency; works identically with JUnit 5 and 6; direct mapping to Scott-CC's pytest `@parametrize`; Kotest/Spek rejected | +| **Apply mechanism** | Full file + diff + manifest, gated by `--auto-approve`; zombie/redundant deletion always requires explicit approval | Matches Scott-CC command contract: non-destructive changes auto-applied; deletions require explicit approval | +| **Data source** | Read pre-computed `redundantGroups` from JSON | Leverages D1's Option C output (Kotlin module already computes groups in `MutationResultsParser`); no duplicated logic; refactor specialist consumes JSON array | + +### Implementation next steps (T2 — straightforward task, no wayfinding needed): +- Update `test-refactor-specialist.md` agent contract: consume `redundantGroups` from JSON, generate full JUnit 5 parameterized test files, produce diff + manifest +- Update `test-quality-reviewer.md` orchestrator: read manifest, gate application on `--auto-approve`, require explicit approval for deletions +- Update `/mutation-test` skill SKILL.md: add `--auto-approve` flag documentation and wiring + +### Frontier after D2 resolution: +- D3 (06-decide-execution-gap-reporting) — blocked by 03 (resolved) → **unblocked and first in order** +## Answer + +**R1 and R2 both resolved.** See `research/01-redundant-test-detection.md` and `research/02-research-auto-refactoring.md`. Ready for grilling. + +## Question + +What is the implementation approach for auto-generated refactored Kotlin test code in OMP's test-refactor-specialist agent? + +### Background + +R1 will research redundant test detection (needed as input for refactoring). R2 will research code generation approaches for Kotlin tests. This ticket decides the overall approach. + +### Task + +1. Review R1 and R2 findings. +2. Decide: + a. **Output format**: Full refactored test file content (like Scott-CC) or incremental patches/edits? + b. **Test framework**: JUnit 5 `@ParameterizedTest` (most likely), Kotest, or Spek? + c. **Apply mechanism**: How does the generated code get applied? Write to file? Git diff? User approval gate? + d. **Dependency on detection**: How does the refactor specialist consume redundant test group detection results from the auditor? + +### Acceptance criteria + +- Chosen output format with rationale +- Chosen test framework with rationale +- Chosen apply mechanism +- Integration approach with the audit pipeline +- Decision recorded as a resolution comment diff --git a/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md b/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md new file mode 100644 index 0000000..1e1ebe2 --- /dev/null +++ b/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md @@ -0,0 +1,54 @@ +Type: grilling +Status: resolved +Blocked by: 03 + +## Answer + +**D3 resolved — execution gap reporting approach.** All 4 decisions confirmed from R3's research: + +| Decision | Choice | Rationale | +|---|---|---| +| **Gap types** | All 5 separate (COMPILATION_FAILURE, IR_TRANSFORMATION_ERROR, BACKSTOP_TIMEOUT, PARTIAL_RUN, NO_OUTPUT) | Separate types enable distinct error messaging and future handling, even if detection overlaps | +| **Detection layers** | Distributed — test-executor + MutationResultsTask/parser | Matches OMP's architecture (executor = capture exit code/missing XML/timeout; parser = analyze build result + XML); most robust for partial-run detection | +| **Score formula** | Null when (total - gaps) == 0 | Mutation score becomes nullable; mirrors Scott-CC's "never manufacture a score" principle; 0.0 implies all survived, misleading when zero evaluations | +| **TimedOut** | NOT a gap — valid result | Mutation was fully evaluated; excluding inflates score | + +**Implementation changes required:** + +- `MutationResults.kt`: Add `gaps: Int`, `mutationsEvaluated: Int` fields; make `mutationScore` nullable (`Double?`); add `ExecutionGap` data class and `execution_gaps: List` field +- `MutationResultsParser.kt`: Add `detectGaps()` function; update `calculateMetrics()` to use new formula +- `.omp/agents/test-executor.md`: Document exit-code + missing XML + timeout checks +- `.omp/mutation-results.gradle.kts`: Check `test` task build result in `generateResults()` + +### Frontier after D3 resolution: +- All 6 straightforward task features (T1: modes, T3: --focus, T4: auto-approve, T6: confidence intervals, T7: gap reporting implementation, T8: rollback) — ready for independent implementation +- The implementation phase begins (no more wayfinding tickets pending) + +## Answer + +**R3 resolved.** See `research/03-research-execution-gap-reporting.md` (346 lines). Ready for grilling. + +## Question + +How should execution gap reporting be adapted for OMP's mutflow-based model? + +### Background + +R3 will research what constitutes an execution gap in mutflow's in-process model (where syntax errors can't occur). This ticket decides the approach. + +### Task + +1. Review R3's findings on mutflow-side gap scenarios. +2. Decide: + a. **Gap definition**: What constitutes an execution gap in OMP? (e.g., compilation failures, IR transformation errors, test class-level failures) + b. **Detection**: Where in the pipeline are gaps detected? (test-executor, Gradle task, parser) + c. **Reporting**: How are gaps excluded from the mutation score denominator? + d. **Output**: Should gaps be in the JSON artifact, the audit report, or both? + +### Acceptance criteria + +- Chosen gap definition +- Chosen detection point in the pipeline +- Chosen exclusion mechanism from score +- Chosen output format +- Decision recorded as a resolution comment diff --git a/.scratch/scott-cc-implementation/map.md b/.scratch/scott-cc-implementation/map.md new file mode 100644 index 0000000..951ed89 --- /dev/null +++ b/.scratch/scott-cc-implementation/map.md @@ -0,0 +1,59 @@ +# Implement 9 Scott-CC→OMP feature gaps in OMP's mutation-testing system + +Labels: wayfinder:map + +## Destination + +**Implement the 9 Scott-CC→OMP feature gaps in OMP's mutation-testing system, shipping each feature independently (incremental delivery). Implementation approaches must be decided for the 3 foggy features (redundant test detection, auto-generated refactoring, execution gap reporting); the remaining 6 are straightforward tasks.** + +Reaches from here when: each of the 9 features has a clear implementation approach and any necessary decisions resolved, the dependency graph is wired, and the frontier (open, unblocked, unclaimed tickets) represents the next implementable units. + +## Notes + +**Source of truth:** [comparison matrix](../scott-cc-comparison/research/02-comparison-matrix.md) — R2's 51-row feature matrix. Domain model: [../scott-cc-comparison/domain-model.md](../scott-cc-comparison/domain-model.md). Original map: [../scott-cc-comparison/map.md](../scott-cc-comparison/map.md). + +**Implementation architecture (domain model):** + +| Component | File(s) | Features to implement here | +|---|---|---| +| `test-auditor` agent | `.omp/agents/test-auditor.md` | Redundant test group detection, execution gap reporting | +| `MutationResultsParser` | `.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt` | Confidence intervals, execution gap reporting | +| `test-refactor-specialist` agent | `.omp/agents/test-refactor-specialist.md` | Auto-generated refactored test code, diff generation, rollback instructions | +| `test-quality-reviewer` agent | `.omp/agents/test-quality-reviewer.md` | Quick/standard/deep modes, `--focus`, `--auto-approve` | +| `/mutation-test` skill | `.omp/skills/mutation-test/SKILL.md` | CLI flags: `--quick`/`--deep`, `--focus`, `--auto-approve` | + +**Dependency graph** (decisions depend on research): +``` +[R1: Redundant test detection] → D1 (decide detection approach, issue 04) +[R2: Auto-refactoring] → D2 (decide refactoring approach, issue 05) +[R3: Execution gap reporting] → D3 (decide gap reporting approach, issue 06) +``` + +**Issue tracker:** local markdown — `.scratch/scott-cc-implementation/` + +**Straightforward tasks (no wayfinding needed — implement after map complete):** +- T1: Quick/standard/deep mode abstraction (M) — map flags to mutflow `maxRuns` +- T3: `--focus` parameter (S) — bridge CLI to Gradle `includeTargets`/`excludeTargets` +- T4: Auto-approve (M) — wire flag to test-refactor-specialist +- T6: Confidence intervals (S) — add statistical CI formula to MutationResultsParser +- T7: Execution gap reporting (S) — adapt gap concept for mutflow (see R3) +- T8: Rollback instructions (trivial) — add to final report + +## Decisions so far + +- [Research R1: redundant test detection](issues/01-research-redundant-test-detection.md): Scott-CC failure-signature algorithm reconstructable from OMP's `testKillerMatrix` + `mutations.killedByTests`. **Option C recommended** — detect in Kotlin module (`MutationResultsParser.detectRedundantTestGroups()`) + describe patterns in test-auditor agent. Threshold >5 (same as Scott-CC). Data model sufficient, no new data capture needed. +- [Research R2: auto-generated refactoring](issues/02-research-auto-refactoring.md): **Full test file content** (not patches) via **JUnit 5 `@ParameterizedTest`** (already a dependency). Apply gated by `--auto-approve`; zombie/redundant deletion always requires explicit approval. Consumes `test_killer_matrix`, `zombie_test_candidates`, `over_mocked_tests` from JSON. +- [Research R3: execution gap reporting](issues/03-research-execution-gap-reporting.md): **5 gap types** at test-class granularity: COMPILATION_FAILURE, IR_TRANSFORMATION_ERROR, BACKSTOP_TIMEOUT, PARTIAL_RUN, NO_OUTPUT. Detection distributed across test-executor (exit code + missing XML) and MutationResultsTask/parser. **TimedOut is NOT a gap** (valid result). New score: `killed/(total-gaps)`, null when denominator is 0. +- [Decided D1: redundant test detection approach](issues/04-decide-redundant-test-detection.md): Option C (both Kotlin module + test-auditor agent). Threshold >5 (same as Scott-CC). Hybrid data source (testKillerMatrix for grouping + mutations.killedByTests for precise output). Output in both JSON + report. User confirmed. +- [Decided D2: auto-refactoring approach](issues/05-decide-auto-refactoring.md): Full test file content via JUnit 5/6 `@ParameterizedTest`. Apply gated by `--auto-approve`; zombie/redundant deletion requires explicit approval. Refactor specialist reads pre-computed `redundantGroups` from JSON (leveraging D1's Option C). User confirmed. +- [Decided D3: execution gap reporting approach](issues/06-decide-execution-gap-reporting.md): 5 gap types (COMPILATION_FAILURE, IR_TRANSFORMATION_ERROR, BACKSTOP_TIMEOUT, PARTIAL_RUN, NO_OUTPUT) at test-class granularity. Distributed detection (test-executor + parser). Score: `null` when `(total - gaps) == 0`; TimedOut is a valid result, not a gap. Add `execution_gaps` array + `mutationsEvaluated` to JSON; make `mutationScore` nullable. User confirmed. + +## Not yet specified + +(none — all design decisions resolved) + +## Out of scope + +- All 9 features are in scope; this is purely the planning map before implementation begins. + +## Status: All decisions resolved (D1, D2, D3). Frontier: 7 straightforward implementation tasks ready for independent shipping. diff --git a/.scratch/scott-cc-implementation/research/01-redundant-test-detection.md b/.scratch/scott-cc-implementation/research/01-redundant-test-detection.md new file mode 100644 index 0000000..622585d --- /dev/null +++ b/.scratch/scott-cc-implementation/research/01-redundant-test-detection.md @@ -0,0 +1,388 @@ +# R1: Research — Redundant Test Group Detection + +**Status:** Research findings — resolved +**Date:** 2026-08-26 +**Author:** ResearchRedundantTestDetection (R1) +**Source files:** Scott-CC test-auditor (`citadelgrad/scott-cc/plugins/mutation-testing/agents/test-auditor.md`), OMP test-auditor (`.omp/agents/test-auditor.md`), OMP typed JSON module (`.omp/mutation-results-src/`), OMP Gradle task (`.omp/mutation-results.gradle.kts`), comparison matrix (`.scratch/scott-cc-comparison/research/02-comparison-matrix.md`) + +--- + +## 1. Executive Summary + +Scott-CC's redundant test group detection uses a **failure-signature algorithm**: each test is assigned a signature (the set of mutation IDs it failed for), tests with identical signatures are grouped, and groups exceeding a threshold of **>5 tests** are flagged as redundant. + +**OMP's data model is sufficient to reconstruct failure signatures.** The `testKillerMatrix` field in OMP's `MutationResults` JSON (`Map>` — test name → mutation source locations it killed) is functionally identical to Scott-CC's failure signature. For higher precision matching Scott-CC's per-mutation granularity, the `mutations` list with `killedByTests` arrays provides a composite key of `(sourceLocation, originalOperator, variantOperator)`. + +**Recommended integration point: both the Kotlin typed module and the test-auditor agent.** The algorithmic grouping computation belongs in `MutationResultsParser` (reliable, unit-testable, produces JSON output); the semantic pattern description and recommendation belong in the test-auditor agent (requires LLM-level understanding). + +--- + +## 2. Scott-CC Failure-Signature Algorithm + +### 2.1 Source + +Scott-CC test-auditor, section "### 3. Find Redundant Test Groups" (lines 106–127 of the source file). + +### 2.2 Algorithm (verbatim from Scott-CC) + +```python +# Algorithm +test_failure_signatures = {} + +for test_name in all_tests: + signature = [] + for result in test_results: + if test_name in result['failures']: + signature.append(result['mutation_id']) + + test_failure_signatures[test_name] = signature + +# Group by signature +from collections import defaultdict +groups = defaultdict(list) + +for test, sig in test_failure_signatures.items(): + groups[tuple(sig)].append(test) + +# If >5 tests have same signature → redundant group +redundant_groups = [tests for sig, tests in groups.items() if len(tests) > 5] +``` + +### 2.3 Step-by-step explanation + +| Step | What happens | Scott-CC data source | +|---|---|---| +| 1 | Enumerate all test names from the intersection of `test_outcomes` keys across all mutation results | `test_results[].test_outcomes` | +| 2 | For each test, build a **failure signature** — the ordered list of mutation IDs the test failed for (i.e., mutations the test caught/killed) | `test_results[].failures` (list of test names that failed per mutation) | +| 3 | Group tests by identical signature (tuple comparison) | In-memory `defaultdict` | +| 4 | Filter groups where `len(tests) > 5` | Threshold: **strictly greater than 5** | + +### 2.4 Key parameters + +| Parameter | Value | +|---|---| +| **Threshold** | >5 tests per group (strictly greater than 5) | +| **Signature type** | Ordered list of mutation IDs (becomes a tuple for grouping key) | +| **Grouping key** | Exact equality of the signature tuple | +| **Empty signatures** | Tests that never failed for any mutation (zombies) get an empty signature — they are **not** flagged as redundant groups (they're caught by the separate zombie detection algorithm) | + +### 2.5 Scott-CC output format for redundant groups + +```json +"redundant_groups": [ + { + "pattern": "Django model field validation", + "tests": ["test_status_valid", "test_status_invalid", "..."], + "count": 150, + "failure_signature": ["mut-003", "mut-007"], + "recommendation": "Consolidate into 1 parameterized test" + } +] +``` + +The `pattern` and `recommendation` fields are **semantically generated** by the LLM auditor (describing the common test pattern and suggesting consolidation). The `tests`, `count`, and `failure_signature` fields are **algorithmically computed**. + +### 2.6 Data flow in Scott-CC + +Scott-CC's data model is **in-memory dict handoff** between agents: + +1. **test-executor agents** (×15, one per mutation) each return a per-mutation result: + ```json + { + "mutation_id": "mut-001", + "test_results": {"total": 200, "passed": 195, "failed": 5}, + "test_outcomes": {"tests/test_stripe.py::test_retry_boundary": "failed"}, + "failures": [{"test": "test_retry_boundary", "error": "..."}] + } + ``` +2. **test-auditor** receives the aggregated list of 15 per-mutation results and reconstructs failure signatures by scanning the `failures` arrays. + +The auditor does **not** receive a pre-computed matrix — it builds signatures from the raw per-mutation `failures` lists by iterating all tests against all mutation results. This is an O(tests × mutations) scan. + +--- + +## 3. OMP Data Model Analysis + +### 3.1 OMP's typed JSON module + +Location: `.omp/mutation-results-src/main/kotlin/io/omp/mutation/` + +``` +MutationResult.kt // Per-mutation result data class +MutationResults.kt // Top-level results container +MutationResultsParser.kt // Pure parsing + matrix building functions +MutationResultsSerializer.kt // JSON serialization +``` + +### 3.2 MutationResult data class + +```kotlin +@Serializable +data class MutationResult( + val sourceLocation: String, // e.g., "(Calculator.kt:7)" + val originalOperator: String, // e.g., ">" + val variantOperator: String, // e.g., ">=" + val result: MutationResultType, // Killed, Survived, TimedOut + val killedByTest: String? = null, // legacy: first killer only + val killedByTests: List = [], // ALL tests that killed this mutation +) +``` + +Key point: `killedByTests` (enabled by the `feature/zombie-detection` fork branch) captures **all** tests that kill each mutation, not just the first. This is the OMP equivalent of Scott-CC's per-mutation `failures` list. + +### 3.3 MutationResults data class + +```kotlin +@Serializable +data class MutationResults( + val generatedAt: Long, + val mutationScore: Double, + val qualityBand: QualityBand, + val confidence: ConfidenceLevel, + val totalMutations: Int, + val killed: Int, + val survived: Int, + val timedOut: Int, + val testMethods: List, // all test method names + val testKillerMatrix: Map>, // test → mutation source locations it killed + val mutations: List, // per-mutation results with killedByTests +) +``` + +### 3.4 How OMP builds the testKillerMatrix + +From `MutationResultsParser.kt`, lines 128–136: + +```kotlin +fun buildTestKillerMatrix(mutations: List): Map> { + val testKillerMatrix = mutableMapOf>() + mutations.forEach { m -> + m.killedByTests.forEach { testName -> + testKillerMatrix.getOrPut(testName) { mutableListOf() }.add(m.sourceLocation) + } + } + return testKillerMatrix +} +``` + +This iterates all killed mutations and, for each killer test, adds the mutation's source location to that test's entry in the map. The result is: **test name → list of mutation source locations it killed**. + +### 3.5 Sufficiency Assessment + +**Verdict: Sufficient.** OMP's data model can fully reconstruct failure signatures. Two approaches are available: + +#### Approach A: Using `testKillerMatrix` directly (simplest) + +The `testKillerMatrix` is functionally identical to Scott-CC's failure signature: + +| Scott-CC | OMP | +|---|---| +| `test_name → [mutation_id, mutation_id, ...]` | `test_name → [sourceLocation, sourceLocation, ...]` | +| mutation_id is unique per mutation | sourceLocation identifies the mutation's location | + +The failure signature for a test is simply `testKillerMatrix[test_name].toSet()`. Grouping is: + +```kotlin +val groups = mutableMapOf, MutableList>() +testKillerMatrix.forEach { (testName, killedLocations) -> + val signature = killedLocations.toSet() + if (signature.isNotEmpty()) { // exclude zombies (empty signature) + groups.getOrPut(signature) { mutableListOf() }.add(testName) + } +} +val redundantGroups = groups.filter { it.value.size > 5 } +``` + +**Limitation**: `sourceLocation` alone does not uniquely identify a mutation if multiple operators are applied at the same line (e.g., `> → >=` and `>= → >` at `(Calculator.kt:7)`). The `testKillerMatrix` would list `Calculator.kt:7` once or twice depending on how many mutations at that location the test killed. For grouping purposes, converting to `Set` collapses duplicates, which means tests that kill different operators at the same line but not identical mutation sets could still be grouped together. + +#### Approach B: Using `mutations` list + `killedByTests` (precise) + +The `mutations` list provides full per-mutation granularity. Each `MutationResult` has a composite key of `(sourceLocation, originalOperator, variantOperator)`, and `killedByTests` lists all killer tests. The failure signature can be reconstructed with full precision: + +```kotlin +val testSignatures = mutableMapOf>() +mutations.forEach { mutation -> + if (mutation.result == MutationResultType.Killed) { + val mutationKey = "${mutation.sourceLocation}:${mutation.originalOperator}->${mutation.variantOperator}" + mutation.killedByTests.forEach { testName -> + testSignatures.getOrPut(testName) { mutableSetOf() }.add(mutationKey) + } + } +} + +val groups = mutableMapOf, MutableList>() +testSignatures.forEach { (testName, signature) -> + if (signature.isNotEmpty()) { + groups.getOrPut(signature.toSet()) { mutableListOf() }.add(testName) + } +} +val redundantGroups = groups.filter { it.value.size > 5 } +``` + +This approach gives exactly Scott-CC's granularity: each mutation is uniquely identified, and the signature captures the full set of mutations each test caught. + +**Trade-off**: Approach A is simpler and uses the already-computed `testKillerMatrix` (no need to re-scan the `mutations` list). Approach B is more precise but requires traversing the `mutations` list. For practical redundancy detection, Approach A is sufficient — the edge case of multiple operators at the same source location producing different test outcomes is rare and would not significantly affect redundancy conclusions. + +#### Approach C: Hybrid (recommended) + +Use `testKillerMatrix` for the signature (it's already computed and in the JSON), but when a redundant group is found, cross-reference with the `mutations` list to provide the precise `failure_signature` in the output (listing the specific mutation operator pairs, not just source locations). + +### 3.6 Scott-CC vs OMP data flow comparison + +| Aspect | Scott-CC | OMP | +|---|---|---| +| **Executor count** | 1 per mutation (15 executors) | 1 per test class (fewer, but mutflow's internal loop handles mutations) | +| **Data handoff** | In-memory dict between agents | Typed Kotlin JSON artifact (`mutation-results.json`) | +| **Per-mutation data** | `failures` list (tests that failed per mutation) | `killedByTests` array (all tests that killed each mutation) | +| **Per-test data** | Reconstructed by auditor from `failures` arrays | Pre-computed `testKillerMatrix` (test → source locations killed) | +| **Mutation identity** | Unique ID (`mut-001`) | Composite key: `(sourceLocation, originalOperator, variantOperator)` | +| **Data sufficiency for signatures** | Yes — `failures` arrays contain all needed info | Yes — `testKillerMatrix` + `mutations.killedByTests` contain all needed info | + +--- + +## 4. Integration Point Recommendation + +### 4.1 Options considered + +| Option | Where | Pros | Cons | +|---|---|---|---| +| **A** | Test-auditor agent only (in-memory from JSON) | No code changes; follows Scott-CC's pattern of auditor-side analysis | LLM agent may produce incorrect set operations on large matrices; not unit-testable; no reusable JSON output | +| **B** | Kotlin module only (Gradle task computes and outputs JSON) | Reliable, unit-testable, follows OMP's typed-module pattern; JSON available to any consumer | Loses LLM-level semantic pattern description; requires JSON schema extension | +| **C** | **Both** (Kotlin module computes groups + agent describes patterns) | Reliable computation + semantic recommendations; follows separation of concerns; JSON available to all consumers | Most work; requires changes in 3 files | + +### 4.2 Recommended: Option C (both) + +#### C.1 Kotlin module changes (`.omp/mutation-results-src/`) + +**File: `MutationResults.kt`** — Add `RedundantGroup` data class and field: + +```kotlin +@Serializable +data class RedundantGroup( + @SerialName("tests") val tests: List, + @SerialName("count") val count: Int, + @SerialName("failureSignature") val failureSignature: List, +) + +// Add to MutationResults: +@SerialName("redundantGroups") val redundantGroups: List = emptyList(), +``` + +**File: `MutationResultsParser.kt`** — Add detection function: + +```kotlin +fun detectRedundantTestGroups( + mutations: List, + threshold: Int = 5 // matches Scott-CC's >5 threshold +): List { + val testSignatures = mutableMapOf>() + + mutations.forEach { mutation -> + if (mutation.result == MutationResultType.Killed) { + val mutationKey = "${mutation.sourceLocation}:${mutation.originalOperator}->${mutation.variantOperator}" + mutation.killedByTests.forEach { testName -> + testSignatures.getOrPut(testName) { mutableSetOf() }.add(mutationKey) + } + } + } + + val groups = mutableMapOf, MutableList>() + testSignatures.forEach { (testName, signature) -> + if (signature.isNotEmpty()) { + groups.getOrPut(signature.toSet()) { mutableListOf() }.add(testName) + } + } + + return groups.filter { it.value.size > threshold } + .map { (signature, tests) -> + RedundantGroup( + tests = tests.sorted(), + count = tests.size, + failureSignature = signature.sorted(), + ) + } + .sortedByDescending { it.count } +} +``` + +**File: `MutationResultsParser.kt`** — Update `assembleResults()` to call the new function. + +**File: `mutation-results.gradle.kts`** — No changes needed (thin adapter already calls `assembleResults()`). + +**Tests**: Add unit tests for `detectRedundantTestGroups()` covering: exact-signature grouping, threshold boundary (5 vs 6 tests), exclusion of zombie tests (empty signature), precision with same-location different-operator mutations, empty input, sorting, and backward compatibility (existing tests still pass). + +#### C.2 Test-auditor agent prompt (`.omp/agents/test-auditor.md`) + +1. **Document the algorithm**: Add a "Find Redundant Test Groups" section describing the failure-signature algorithm, threshold (>5), and how it reads `redundantGroups` from the JSON. +2. **Read pre-computed data**: The agent reads `redundantGroups` from `mutation-results.json` (computed by the Kotlin module). +3. **Generate semantic output**: For each group, the agent generates: + - `pattern`: A human-readable description of the common test pattern (e.g., "Django model field validation") + - `recommendation`: Concrete suggestion (e.g., "Consolidate into 1 parameterized test") +4. **Include in output report**: Add `redundant_groups` to the auditor's JSON output. + +#### C.3 Rationale for splitting between Kotlin and agent + +OMP already follows this separation for zombie detection: the `testKillerMatrix` is **computed** in the Kotlin module (data preparation), while the **detection** (finding tests not in the matrix) is done by the agent. For redundant groups, the split is: + +- **Kotlin module**: Algorithmic grouping computation (set operations, threshold filtering, sorting) — deterministic, unit-testable, language-appropriate. +- **Test-auditor agent**: Semantic pattern description and recommendation generation — requires understanding the test source code, naming conventions, and context. + +This is cleaner than Scott-CC's approach, where the auditor does everything in Python (including the set grouping, which is error-prone for an LLM on large matrices). OMP's typed module provides reliability for the computational part, while the LLM handles what it does best: semantic reasoning. + +### 4.4 Backward compatibility + +- Adding `redundantGroups` with a default of `emptyList()` to `MutationResults` is backward-compatible: existing JSON consumers that don't use the field will ignore it, and `encodeDefaults = true` ensures it appears in output. +- The field name follows OMP's existing camelCase convention (`testKillerMatrix`, `mutationScore`, etc.). + +--- + +## 5. Gap Summary + +| Scott-CC feature | OMP current state | What's needed | +|---|---|---| +| Failure-signature algorithm (>5 threshold) | Not present | Add `detectRedundantTestGroups()` to `MutationResultsParser` | +| `redundant_groups` in audit output | Not present | Add `RedundantGroup` data class + field to `MutationResults`, include in JSON | +| Pattern/recommendation generation | Not present (no redundant detection at all) | Document algorithm in test-auditor agent prompt; agent generates `pattern` + `recommendation` | +| Unit tests for the algorithm | N/A | Add tests in `MutationResultsParserTest.kt` | + +**Effort estimate**: S–M (small-medium). The algorithm is ~25 lines of Kotlin. The main work is adding the data class, the function, updating `assembleResults()`, and writing unit tests. The agent prompt update is trivial. + +--- + +## 6. Algorithm Specification for Implementation + +``` +INPUT: + mutations: List // each has sourceLocation, originalOperator, + // variantOperator, result, killedByTests + threshold: Int = 5 // Scott-CC uses >5 (strictly greater than) + +OUTPUT: + List sorted by count descending, each containing: + - tests: sorted list of test names sharing the signature + - count: number of tests in the group + - failureSignature: sorted list of mutation key strings + - (pattern and recommendation added by test-auditor agent) + +ALGORITHM: + 1. For each mutation where result == Killed: + a. Compute mutation_key = "${sourceLocation}:${originalOperator}->${variantOperator}" + b. For each testName in mutation.killedByTests: + - Add mutation_key to testSignatures[testName] + 2. For each test in testSignatures: + a. signature = testSignatures[testName].toSet() + b. If signature is non-empty (exclude zombies): + - Add testName to groups[signature] + 3. For each group where len(tests) > threshold: + - Create RedundantGroup(tests=sorted(tests), count=len(tests), + failureSignature=sorted(signature)) + 4. Sort results by count descending + 5. Return list + +EDGE CASES: + - Empty input → empty list + - No killed mutations → empty list + - All tests have empty signatures (all zombies) → empty list (no redundant groups) + - Multiple operators at same source location → distinguished by operator in key + - Ties in count → sorted alphabetically by first test name +``` diff --git a/.scratch/scott-cc-implementation/research/02-research-auto-refactoring.md b/.scratch/scott-cc-implementation/research/02-research-auto-refactoring.md new file mode 100644 index 0000000..525ac5a --- /dev/null +++ b/.scratch/scott-cc-implementation/research/02-research-auto-refactoring.md @@ -0,0 +1,366 @@ +# R2 Research: Auto-Generated Refactored Kotlin Test Code + +**Status:** Research findings — resolved +**Date:** 2026-08-26 +**Author:** ResearchAutoRefactoring (R2) +**Source files:** Scott-CC `plugins/mutation-testing/agents/test-refactor-specialist.md`, `.omp/agents/test-refactor-specialist.md`, `.omp/agents/test-auditor.md`, `.omp/agents/test-executor.md`, `.omp/agents/test-quality-reviewer.md`, `.omp/skills/mutation-test/SKILL.md`, `plugins/mutation-testing/commands/mutation-test.md`, `plugins/mutation-testing/agents/test-auditor.md`, `plugins/mutation-testing/agents/test-quality-reviewer.md`, `.omp/mutation-results-src/` (buildSrc typed module), `.scratch/scott-cc-comparison/research/02-comparison-matrix.md`, `.scratch/scott-cc-comparison/domain-model.md` + +--- + +## 1. Summary + +Scott-CC and OMP both conceptualize the same four refactoring actions (consolidate redundant tests, remove zombies, add edge cases, replace over-mocked with integration tests). The gap is **execution, not category**: Scott-CC auto-generates production-ready full test files and can auto-apply them; OMP's test-refactor-specialist only produces suggestions. The OMP agent description itself already calls for "full refactored test file content" — the gap is that no code-generation mechanism or pipeline wiring has been implemented. + +The key technical decision is straightforward: **JUnit 5 `@ParameterizedTest`** is the correct Kotlin test framework (it is already a dependency in OMP's bootstrap), **full file content** is the recommended output format (matches the agent contract and Scott-CC's proven approach), and the consume path from the test-auditor's `testKillerMatrix` and `surviving_mutations` is well-defined but requires redundancy derivation logic that Scott-CC's auditor produces natively. + +--- + +## 2. Scott-CC: How Full Test File Generation Works + +### 2.1 Workflow (5 steps) + +1. **Read existing test file** — the agent reads the test source to understand framework, imports, fixtures, and style. +2. **Analyze structure** — identify test framework, naming conventions, shared patterns. +3. **Generate refactored code** — for each action: consolidate (extract common pattern → parameterized test), remove (zombie with diff + explanation), add (edge-case tests following existing style), replace over-mocked (swap mocks for integration tests). +4. **Create complete refactored file** — emit the **entire** test file as a single output, with a module-level docstring documenting changes, metrics, and rationale. +5. **Generate git diff** — produce a full before/after diff so the user can review deletions, consolidations, and additions. + +### 2.2 Output JSON Contract + +```json +{ + "refactored_test_code": "... full test file ...", + "changes": { + "removed": ["test_name", ...], + "consolidated": [{"from": ["test_a", ...], "to": "test_fn", "type": "parameterized"}], + "added": ["test_boundary_at_3", ...] + }, + "metrics": { + "old_test_count": 200, + "new_test_count": 20, + "estimated_mutation_score": 0.85 + }, + "diff": "... git diff ...", + "recommendations": ["..."], + "warnings": ["..."] +} +``` + +### 2.3 Estimation Formulas + +**Mutation score improvement** (conservative): each new edge-case test catches ~1.5 additional mutations; estimated score = (currentCaught + newTests × 1.5) / totalMutations. + +**Execution time reduction**: parameterized tests share setup/teardown overhead. old_time = oldCount × (avgSetup + avgTest); new_time = newCount × (avgSetup + avgTest); speedup = old_time / new_time. + +### 2.4 Auto-Apply: `--auto-approve` + +The `/mutation-test` command (command contract, `plugins/mutation-testing/commands/mutation-test.md`) passes `auto_approve` to the orchestrator. When `--auto-approve` is present, the orchestrator applies the refactoring proposal without a second confirmation. **Critically, `--auto-approve` never permits deleting tests that the audit did not classify as zombie or redundant** — zombie deletion always requires explicit approval. + +### 2.5 Framework-Specific Patterns (Python/JS) + +| Framework | Pattern | Example | +|---|---|---| +| pytest | `@pytest.mark.parametrize("field,value,expected", [...])` | `@pytest.mark.parametrize("status", ["active", "canceled", ...])` | +| unittest | `@parameterized.expand([...])` | Decorator-based, class must extend `unittest.TestCase` | +| Jest/Vitest | `describe.each([{...}, ...])("label", ({status, expected}) => {...})` | Callback-based table-driven | + +--- + +## 3. OMP: Current State (Suggestions Only) + +### 3.1 Agent Contract + +The OMP `test-refactor-specialist.md` (agent contract, `.omp/agents/test-refactor-specialist.md`) already declares the output format as **"Return the full refactored test file content"** plus a list of changes and rationale. This means the *intent* matches Scott-CC — the gap is that no code-generation mechanism is wired into the pipeline. The agent runs as the final phase of the `test-quality-reviewer` orchestrator. + +### 3.2 Constraints + +- Does NOT run tests (test-executor's job) +- Does NOT modify production source (test files only) +- Does NOT create mutations (test-saboteur's job) +- Focus on mutated classes identified by the auditor + +### 3.3 Refactoring Actions (same categories as Scott-CC) + +1. **Zombie test candidates** — review each: false positive (doesn't exercise mutated path) vs true zombie (should have caught but didn't). +2. **Over-mocked tests** (>3 mocks) — can mocks be replaced with real implementations? +3. **Surviving mutations** — identify which test SHOULD have caught it; add boundary/edge/negation tests. +4. **Consolidate redundant tests** — if multiple tests cover the same path, consolidate + add missed edge cases. +5. **Add edge cases** — Scott-CC's 5 mutation strategies mapped to test improvements (boundary, return values, boolean logic, arithmetic). + +### 3.4 No Code Generation Yet + +Comparison matrix §3.1 (row 3.1) confirms: **Output type** — Scott-CC produces production-ready full file content; OMP produces "suggestions only — no code auto-generation." Gap direction: Scott-CC-only. Effort to close: **L** (code generator + apply mechanism). + +Missing sub-features per the matrix: + +| Matrix Row | Scott-CC | OMP | Gap | Effort | +|---|---|---|---|---| +| 3.1 | Full file content | Suggestions only | Scott-CC-only | L | +| 3.2 | `--auto-approve` | No auto-apply | Scott-CC-only | M | +| 3.3 | Same action categories | Same categories | Both-have | N/A | +| 3.4 | Full git diff | No diff | Scott-CC-only | S | +| 3.5 | Metrics estimation | No metrics | Scott-CC-only | M | +| 3.6 | Language-specific patterns | Kotlin patterns | Both-have | N/A | +| 3.7 | AskUserQuestion approval | Manual apply gate | Both-have | N/A | + +--- + +## 4. Kotlin Test Framework Analysis + +### 4.1 OMP's Existing Test Stack + +The OMP project's only Kotlin test files (in `buildSrc` — `.omp/mutation-results-src/test/kotlin/io/omp/mutation/`) use **JUnit 5 (Jupiter)**: + +```kotlin +// From MutationResultsParserTest.kt +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Assertions.* +``` + +The `buildSrc/build.gradle.kts` configures: +```kotlin +testImplementation("org.junit.jupiter:junit-jupiter-api:6.1.3") +testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:6.1.3") +testRuntimeOnly("org.junit.platform:junit-platform-launcher:6.1.3") +tasks.test { useJUnitPlatform() } +``` + +The `test-executor` agent confirms OMP uses JUnit 6 (`JUnit 6 extension handles the multi-run model internally`), and the `bootstrap-mutation-testing.sh` script installs JUnit 6 dependencies into the target project's `build.gradle.kts`. + +### 4.2 Available Kotlin Test Frameworks + +| Framework | Pros | Cons | OMP Status | +|---|---|---|---| +| **JUnit 5 `@ParameterizedTest`** | Already a dependency (JUnit 5/6 installed by bootstrap). `@ParameterizedTest` + `@CsvSource` / `@MethodSource` / `@ValueSource` / `@EnumSource` provide full parameterization. Mature, well-documented, Kotlin-friendly. Direct equivalent of pytest `@parametrize` and Jest `describe.each`. | Some verbosity with `@MethodSource` (need factory methods). | ✅ Ready — no new deps | +| **Kotest** | Idiomatic Kotlin DSL (`table`, `row`). `StringSpec`, `FunSpec`, `BehaviorSpec` with built-in parameterization. | Requires adding `kotest-runner-junit5` dependency. Not in OMP's bootstrap. Different test structure paradigm — existing JUnit 5 tests wouldn't mix naturally. | ❌ Not installed | +| **Spek** | Behavior-driven (given/when/then). | Not actively maintained (last release 2021). No native parameterized test support. Requires new dependency. | ❌ Not installed | + +### 4.3 Recommendation: JUnit 5 `@ParameterizedTest` + +**Rationale:** + +1. **No new dependencies** — JUnit 5/6 is already installed by the bootstrap script and used in all existing OMP test files. Kotest and Spek would require new dependencies, new test runner configuration, and a paradigm shift. +2. **Direct conceptual mapping** — `@ParameterizedTest` with `@CsvSource` maps cleanly to Scott-CC's pytest `@parametrize` pattern: + - pytest: `@pytest.mark.parametrize("status", ["active", "canceled", "trialing"])` + - JUnit 5: `@ParameterizedTest @EnumSource` or `@CsvSource` variant + - pytest: `@pytest.mark.parametrize("field,value,expected", [("status","active",True), ...])` + - JUnit 5: `@ParameterizedTest @CsvSource("status,active,true", "status,canceled,true", ...)` +3. **Kotlin interoperability** — JUnit 5 annotations work natively in Kotlin. `@ParameterizedTest` + `@MethodSource` is the most Kotlin-idiomatic (can return `Stream` or use `@CsvSource` for simpler cases). +4. **Consistency** — all existing OMP tests use JUnit 5 Jupiter. Mixing Kotest or Spek would fragment the test suite. + +**Kotlin JUnit 5 parameterization patterns:** + +```kotlin +// Simple value source (like pytest's parametrize with single arg) +@ParameterizedTest +@ValueSource(strings = ["active", "canceled", "trialing", "past_due", "unpaid"]) +fun `subscription status validation`(status: String) { + val sub = Subscription(status = status) + assertEquals(status, sub.status) +} + +// CSV source (multiple parameters, like pytest table) +@ParameterizedTest +@CsvSource( + "0, false", + "1, false", + "2, false", + "3, true", // Boundary — caught mut-001 (>= 3 → > 3) + "4, true", + "5, true", +) +fun `retry count boundary logic`(retryCount: Int, shouldRaise: Boolean) { + if (shouldRaise) { + assertThrows(MaxRetriesExceeded::class.java) { + processPayment(retryCount = retryCount) + } + } else { + assertTrue(processPayment(retryCount = retryCount).success) + } +} + +// Method source for complex objects +@ParameterizedTest +@MethodSource("boundaryCases") +fun `boundary condition test`(input: Int, expected: Boolean) { + assertEquals(expected, isPositive(input)) +} + +companion object { + @JvmStatic + fun boundaryCases() = Stream.of( + Arguments.of(0, false), + Arguments.of(1, true), + Arguments.of(-1, false), + ) +} +``` + +--- + +## 5. Recommended Output Format + +### 5.1 Full File Content (not incremental patches) + +**Recommendation:** Generate the **full refactored test file content**, following Scott-CC's proven approach. + +**Rationale:** + +1. **Agent contract already specifies it** — the OMP `test-refactor-specialist.md` says "Return the full refactored test file content." This aligns with Scott-CC. +2. **Kotlin's type safety makes incremental patches fragile** — patching Kotlin requires maintaining type-correct imports, proper `@ParameterizedTest`/`@MethodSource` factory methods, and companion object structure. A full file regeneration avoids partial-state errors. +3. **LLM agents generate full files naturally** — the test-refactor-specialist is an LLM that can read the original test file and emit a complete replacement. This is the same pattern Scott-CC uses. +4. **Diff can be derived** — once the full file is generated, a git diff is trivially produced by writing to a temp file and running `git diff`. + +### 5.2 Structured Output Contract (matching Scott-CC) + +The test-refactor-specialist should return: + +```json +{ + "refactored_test_code": "... full Kotlin test file ...", + "changes": { + "removed": [{"test": "testName", "file": "path", "line": 47, "reason": "zombie — never caught any mutation"}], + "consolidated": [{"from": ["testA", "testB", ...], "to": "testParameterized", "type": "parameterized", "count": N}], + "added": [{"test": "testBoundaryAt3", "mutation_location": "(Calculator.kt:7)", "mutation_caught": "> → >=", "rationale": "..."}] + }, + "metrics": { + "old_test_count": 200, + "new_test_count": 20, + "reduction_percentage": 90, + "old_mutation_score": 0.23, + "estimated_new_mutation_score": 0.85 + }, + "diff": "... git diff ..." +} +``` + +### 5.3 Diff Generation Approach + +Since the test-refactor-specialist operates as an agent in the OMP harness (not via Claude Code's Edit tool), diff generation should be handled by: + +1. The agent writes the refactored file content to a temporary path. +2. A `git diff` between the original and temp file produces the diff. +3. The diff is included in the structured output for the user to review. + +This mirrors Scott-CC's `diff` field in the output JSON (matrix §3.4, row 3.4). + +--- + +## 6. Integration Approach: Consuming Audit Results + +### 6.1 What the Test-Auditor Produces (OMP) + +The OMP `test-auditor.md` (`.omp/agents/test-auditor.md`) outputs a JSON report with: + +| Field | Type | Description | +|---|---|---| +| `mutation_score` | Double | killed / total | +| `quality_band` | Enum | Excellent/Good/Fair/Poor | +| `confidence` | Enum | Low/Medium/High | +| `total_mutations` | Int | | +| `killed` / `survived` / `timed_out` | Int | | +| `surviving_mutations` | List[String] | e.g. `"(Calculator.kt:5) > → >="` | +| `zombie_test_candidates` | List[String] | Test method names | +| `over_mocked_tests` | List[Object] | `{"method": "...", "mock_count": N}` | +| `test_killer_matrix` | Map> | test name → mutation source locations killed | +| `recommendations` | List[String] | | + +### 6.2 What Scott-CC's Auditor Produces (for comparison) + +Scott-CC's test-auditor (`.mp/agents/test-auditor.md`, lines 1–300+) adds two fields that OMP's auditor lacks: + +- **`redundant_groups`** — tests that always fail together (same `failure_signature`). Each group includes the test list, count, and recommendation. This is the **primary input** for the consolidation refactoring action. +- **`missing_coverage`** — per surviving mutation: type (boundary/return/boolean/arithmetic), line number, original code, suggestion, and which mutation survived. This feeds directly into the "add edge case tests" action. + +### 6.3 How OMP's Refactor Specialist Must Consume Audit Results + +#### 6.3.1 Deriving Redundant Groups from `test_killer_matrix` + +OMP's auditor does **not** output `redundant_groups` directly. Instead, the refactor specialist must derive redundancy from the `test_killer_matrix`: + +``` +Algorithm: +1. For each test in test_killer_matrix, its "signature" = the set of mutation source locations it killed. +2. Group tests whose signatures are identical (or one is a subset of another's). +3. Groups with >5 tests (matching Scott-CC's threshold) → redundant groups eligible for consolidation. +4. For each group, recommend consolidating into a single parameterized test, preserving the union of mutation-killing coverage. +``` + +This is O(n²) for signature comparison (n = number of test methods). For typical projects with 50–200 tests, this is trivial. The `test_killer_matrix` already contains exactly the data needed — Scott-CC derives the same thing from `test_outcomes` per mutation, but the matrix is the inverse view and equally sufficient. + +**Dependency on R1:** The `feature/redundant-test-groups` research (R1) investigated implementing this detection *in the auditor*. If R1's recommendation is to add `redundant_groups` to the auditor's JSON output, the refactor specialist receives it pre-computed. If not, the refactor specialist derives it from `test_killer_matrix` (always available). Either way, the refactor specialist needs the matrix. + +#### 6.3.2 Cross-Referencing Zombies with Surviving Mutations + +OMP's `zombie_test_candidates` is a flat list of test names. To generate targeted improvements: + +``` +For each zombie test candidate: +1. Read the test source file to find the test method. +2. Read the production source to find what code it exercises. +3. Cross-reference with surviving_mutations: find mutations whose source locations + fall within the code paths the zombie test *should* have exercised. +4. Generate a strengthened version of the test with stronger assertions or edge-case inputs. +``` + +Scott-CC's auditor provides richer context (`mutations_it_should_have_caught` per zombie), but OMP's refactor specialist can derive the same by reading source files — which it must do anyway to generate Kotlin test code. + +#### 6.3.3 Consuming Over-Mocked Tests + +OMP's `over_mocked_tests` (`{"method": "...", "mock_count": N}`) maps directly to Scott-CC's. The refactor specialist reads the test file, identifies the `mockk()`/`@MockK`/Mockito `mock()` calls (threshold: >3 per matrix §2.6 row 2.6), and generates an integration-test variant using real implementations where feasible. + +#### 6.3.4 Consuming Surviving Mutations + +OMP's `surviving_mutations` (e.g., `"(Calculator.kt:5) > → >="`) gives source location + operator change. The refactor specialist: + +1. Reads the production source at that location to understand the logic. +2. Generates a Kotlin test that exercises the boundary/mutated path. +3. For `>` → `>=` mutations, adds a test at the exact boundary value. +4. For `<` → `<=` mutations, adds the symmetric boundary test. + +This mirrors Scott-CC's `missing_coverage` with `type: boundary_condition` and `suggestion` fields. + +--- + +## 7. Recommended Apply Mechanism + +### 7.1 Output Contract to Orchestrator + +The test-refactor-specialist should **output the full refactored file content** (not write to the production test file directly). The `test-quality-reviewer` orchestrator then: + +1. Presents the full refactored file + diff + changes manifest to the user. +2. If `--auto-approve` is set (planned as T4, matrix §4.3 row 8.3): apply without second confirmation, **but never auto-delete zombie/redundant tests without approval** (Scott-CC's safety principle, command contract lines 52–55). +3. If not auto-approved: ask the user to accept or refuse. Only on acceptance does the orchestrator write the file. + +### 7.2 Writing the Refactored File + +When the user/orchestrator approves, the refactored test file is written using the `write` tool (full file content). The test-refactor-specialist agent has `write` in its tools list (`.omp/agents/test-refactor-specialist.md` line 4), so it can write directly if the orchestrator delegates write authority. + +### 7.3 Post-Apply Verification + +Per OMP's constraints, the test-refactor-specialist does NOT run tests. After refactoring is applied: + +- The `test-quality-reviewer` orchestrator should dispatch `test-executor` agents to re-run mutation testing on the refactored test class. +- This verifies the estimated mutation score improvement (from §5.2 metrics). + +--- + +## 8. Key Findings Summary + +| Decision Point | Recommendation | Rationale | +|---|---|---| +| **Output format** | Full refactored test file content | (1) Agent contract already specifies it; (2) Kotlin type safety makes incremental patches fragile; (3) matches Scott-CC's proven approach; (4) diff is trivially derived | +| **Test framework** | JUnit 5 `@ParameterizedTest` | Already a dependency (JUnit 5/6 in bootstrap); no new deps; direct mapping to pytest `@parametrize` and Jest `describe.each`; consistent with existing OMP tests | +| **Apply mechanism** | Agent outputs full file + diff + manifest; orchestrator gates via `--auto-approve` | Separates generation from application; `--auto-approve` controls auto-apply; zombie/redundant deletion always requires explicit approval (Scott-CC principle) | +| **Consume audit results** | Derive redundant groups from `test_killer_matrix`; cross-ref zombies with `surviving_mutations`; read source files for context | OMP's auditor doesn't output `redundant_groups` or `missing_coverage` directly, but `test_killer_matrix` + `surviving_mutations` provide equivalent data; source reading is needed anyway for Kotlin code generation | + +### 8.1 Dependencies on Other Research Tickets + +- **R1 (redundant test detection):** If R1 recommends adding `redundant_groups` to the auditor's JSON output, the refactor specialist receives it pre-computed and skips derivation from `test_killer_matrix`. If R1 recommends NOT adding it, the refactor specialist derives it (algorithm in §6.3.1). Either way, the refactor specialist depends on the `test_killer_matrix` being present in the audit JSON. +- **R3 (execution gap reporting):** The refactor specialist should not refactor test classes flagged with execution gaps (ERROR/INVALID_MUTATION from executor). These gaps indicate environment issues, not test quality issues. The `test-executor.md` (OMP) notes mutflow's JUnit extension handles multi-run internally; execution gaps in OMP's model map to executor timeouts or Gradle failures, not Scott-CC's worktree/syntax-error gaps. + +### 8.2 Remaining Questions for the Decision Ticket (05-decide-auto-refactoring) + +1. Should the refactor specialist write the refactored file directly (via `write` tool), or should it return content for the orchestrator to apply? — **Default: return content; let orchestrator handle writes** (matches Scott-CC's `refactored_test_code` field + `--auto-approve` contract). +2. Should `redundant_groups` be added to the auditor's JSON output (requires R1's decision), or derived in the refactor specialist? — **Default: derive from `test_killer_matrix`; upgrade to auditor output if R1 recommends it.** +3. Should metrics estimation use Scott-CC's formulas verbatim? — **Yes** for test count reduction; mutation score estimation should be conservative (§2.3). diff --git a/.scratch/scott-cc-implementation/research/03-research-execution-gap-reporting.md b/.scratch/scott-cc-implementation/research/03-research-execution-gap-reporting.md new file mode 100644 index 0000000..3934528 --- /dev/null +++ b/.scratch/scott-cc-implementation/research/03-research-execution-gap-reporting.md @@ -0,0 +1,346 @@ +# R3 Research: Execution Gap Reporting — Adapting Scott-CC's Model for OMP/mutflow + +**Status:** Research findings — resolved +**Date:** 2026-08-26 +**Author:** ResearchExecutionGapReporting (R3) +**Source files:** +- Scott-CC: `citadelgrad/scott-cc/plugins/mutation-testing/agents/test-auditor.md` (lines 73-81 score formula; lines 192-254 `execution_gaps` handling) +- Scott-CC: `citadelgrad/scott-cc/plugins/mutation-testing/agents/test-executor.md` (lines 200-219 ERROR/INVALID_MUTATION output) +- OMP: `.omp/agents/test-executor.md` (lines 1-46) +- OMP: `.omp/agents/test-auditor.md` (lines 9-61) +- OMP: `.omp/mutation-results.gradle.kts` (lines 1-112, full file) +- OMP: `.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt` (lines 29-163) +- OMP: `.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResults.kt` (lines 14-61) +- OMP: `.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResult.kt` (lines 14-30) +- OMP: `.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsSerializer.kt` (lines 13-28) +- Comparison matrix: `.scratch/scott-cc-comparison/research/02-comparison-matrix.md` (rows 2.9, 7.1-7.7) +- OMP domain model: `.scratch/scott-cc-comparison/domain-model.md` (lines 38-53) +- R3 issue ticket: `.scratch/scott-cc-implementation/issues/03-research-execution-gap-reporting.md` + +**See also:** `.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md` (D3 decision ticket, blocked by this research) + +--- + +## 1. Executive Summary + +Scott-CC reports execution gaps for two per-mutation failure modes — `ERROR` (test suite could not execute) and `INVALID_MUTATION` (the mutation itself introduced a syntax error) — and excludes both from the mutation score denominator. These are possible because Scott-CC uses one isolated git worktree per mutation, so each worktree's test run can independently fail due to environment issues or bad mutations. + +OMP/mutflow does not produce these failure modes. mutflow injects mutations at the Kotlin **IR level at compile time** (compile-once meta-mutant, see comparison matrix 1.1-6.5). All mutations are compiled together into a single binary; at runtime, mutflow's JUnit 6 extension selects one active mutation per run and runs the test suite. mutflow's `MutationResultType` enum has exactly three values: `Killed`, `Survived`, `TimedOut` — no `ERROR` or `INVALID_MUTATION`. Syntax errors from mutations cannot occur at runtime because they would fail at compile time first, failing the entire Gradle `test` task for the whole test class (not a single mutation). mutflow's `TestExecutionExceptionHandler` also swallows test failures during mutation runs, so JUnit always sees "passed" — there is no per-mutation "error" state. + +However, OMP-side scenarios **can** produce gaps that affect mutation evaluation: **compilation failures**, **IR transformation errors**, **backstop timeouts** (15-minute OMP limit), and **partial/truncated runs**. These occur at the test-class granularity (not per-mutation), because mutflow's architecture compiles all mutations for a test class together. The adaptation: shift gap detection from per-mutation executor-agent status to the Gradle task / parser level, where compilation failures and truncated output can be detected, and represent the entire affected test-class's mutation set as gaps. + +**Key finding on TimedOut:** A `TimedOut` result is **not** an execution gap. The mutation was fully evaluated — it caused an infinite loop and mutflow detected it. It should remain in the score denominator as a valid (non-killed) result. The wayfinder map's "Not yet specified" question (whether TimedOut counts as a gap) is resolved: it does not. + +--- + +## 2. Scott-CC Gap Analysis + +### 2.1 What Scott-CC tracks as execution gaps + +Scott-CC's test-auditor (test-auditor.md, Output Format section) explicitly handles two gap-producing statuses from the test-executor agents: + +**ERROR** — The test suite failed to execute at all. The executor returns: +```json +{ + "mutation_id": "mut-003", + "error": "ModuleNotFoundError: No module named 'stripe'", + "status": "ERROR", + "recommendation": "Check dependencies in worktree" +} +``` +This is an environment/infrastructure failure: missing dependencies, import errors, or other issues preventing the test suite from running. + +**INVALID_MUTATION** — The mutation itself introduced a syntax error. The executor returns: +```json +{ + "mutation_id": "mut-005", + "error": "SyntaxError: invalid syntax (stripe_handler.py, line 47)", + "status": "INVALID_MUTATION", + "recommendation": "Saboteur created invalid mutation - skip this one" +} +``` +This means the LLM-guided saboteur produced a mutation that breaks the source code's syntax. + +Both are captured in the auditor's output JSON in the `execution_gaps` array: +```json +"execution_gaps": [ + {"mutation_id": "...", "status": "...", "reason": "..."} +] +``` + +### 2.2 How gaps affect the score + +The auditor's mutation score calculation (test-auditor.md, lines 75-79) explicitly excludes ERROR/INVALID_MUTATION results: + +```python +executable = [result for result in results if result['status'] == 'COMPLETED'] +mutations_caught = count(executable where test_results.failed > 0) +mutations_survived = count(executable where test_results.failed == 0) +mutation_score = mutations_caught / len(executable) +``` + +Key design decisions: +- Only `status == 'COMPLETED'` results are counted as `executable` — the score denominator is the count of executable mutations. +- `mutations_evaluated` = `len(executable)` — reported in the summary with a stated reduced sample size when gaps exist. +- If `mutations_evaluated` is zero (all results were ERROR/INVALID_MUTATION), `mutation_score` and `quality_rating` are set to `null` — the auditor **never manufactures a score**. +- Gap entries include `mutation_id`, `status`, and `reason`, providing traceability. + +### 2.3 Why Scott-CC tracks these gaps + +Scott-CC's isolation model is **one git worktree per mutation** (comparison matrix rows 6.1-6.5). Each test-executor agent runs `pytest`/`npm test` in its own isolated worktree. This per-mutation granularity means: +- Each worktree can independently fail due to environment issues (ERROR) +- Each mutation can independently introduce a syntax error (INVALID_MUTATION) +- The saboteur is LLM-guided, so it may create semantically invalid mutations + +The gap tracking is necessary because each mutation is an independent evaluation. A gap means "we could not evaluate this mutation, so it must not count in the score." Without gap exclusion, a syntax error in one mutation would depress the score unfairly. This is also consistent with the "Error Handling" section at the end of test-auditor.md (lines 376-387): "If test results are missing: report which mutations lack results; calculate partial score with caveat." + +--- + +## 3. mutflow's Model (OMP) + +### 3.1 What mutflow produces + +mutflow's `MutationResultType` enum (MutationResult.kt, lines 26-30) has exactly three values: +```kotlin +enum class MutationResultType { + @SerialName("Killed") Killed, + @SerialName("Survived") Survived, + @SerialName("TimedOut") TimedOut, +} +``` + +The OMP parser (MutationResultsParser.kt, lines 48-53) maps mutflow's console status icons to these types: +- `✓` → `Killed` (at least one test failure caught the mutation) +- `✗` → `Survived` (all tests passed — mutation not caught) +- `⏱` → `TimedOut` (infinite-loop mutation detected by mutflow's 60s internal timeout) + +The parser's `else -> continue` (line 52) skips any unrecognized status icons, meaning only these three types are recognized. + +### 3.2 Why ERROR/INVALID_MUTATION cannot occur + +**No syntax errors from mutations:** mutflow operates at the Kotlin IR level (comparison matrix 1.16-1.17, 7.1). Mutations are IR branch injections applied during compilation — the "compile-once meta-mutant" model (domain-model.md, lines 30, 84-87). All mutations are compiled together into one binary. If a mutation produced invalid IR, it would fail at compile time. The Gradle `test` task would fail entirely, producing no mutflow summary output at all. There is no per-mutation "invalid mutation" result because all mutations share a single compilation step. + +**No environment isolation issues:** mutflow runs all mutations in-process via JUnit 6 extension (test-executor.md, line 17; SKILL.md, line 50-52). The test environment is set up once during the baseline run (run 0). There is no per-mutation environment setup that could fail independently. Dependencies are resolved at build time; if they're missing, the entire `test` task fails, not individual mutations. + +**Test execution exception swallowing:** mutflow's JUnit 6 extension uses a `TestExecutionExceptionHandler` that catches and swallows test failures during mutation runs (test-executor.md, line 32). From JUnit's perspective, all tests "pass" during mutation runs. There is no "error" state per mutation — either a test catches the mutation (Killed) or it doesn't (Survived). The only failure mode is timeout, which mutflow handles and reports as `TimedOut`. + +### 3.3 OMP's current score calculation + +The OMP parser's `calculateMetrics` (MutationResultsParser.kt, lines 93-122) does NOT implement gap exclusion: +```kotlin +val total = mutations.size +val killed = mutations.count { it.result == MutationResultType.Killed } +val survived = mutations.count { it.result == MutationResultType.Survived } +val timedOut = mutations.count { it.result == MutationResultType.TimedOut } +val score = if (total > 0) killed.toDouble() / total else 0.0 +``` + +Score = `killed / total` — all parsed mutation results are counted in the denominator. There is no concept of "executable" vs. "gap" mutations. The comparison matrix (row 2.9) confirms this: OMP has no execution gap reporting. + +--- + +## 4. OMP-Side Gap Scenarios + +While mutflow's three-result model prevents per-mutation gaps, several infrastructure-level failure modes in OMP's pipeline can produce gaps where mutations are not fully evaluated. These operate at the **test-class granularity** (all mutations for a test class share a single Gradle build), not per-mutation. + +### 4.1 Compilation failure (most likely gap scenario) + +mutflow injects mutations at compile time via IR transformation (comparison matrix 1.16). If the IR transformation produces code that fails to compile (e.g., type mismatch, nullability issue, incompatible branch types), the Gradle `compileTestKotlin` (or `compileKotlin`) task fails. The `test` task never starts, and consequently the `mutationResults` task — which `dependsOn(tasks.matching { it.name == "test" })` (mutation-results.gradle.kts, line 48) — never runs either. No JUnit XML is produced, no `MutationTestingSummary` is printed, and no JSON artifact is generated. + +**Scope impact:** All mutations for the affected test class are gaps. We cannot distinguish which specific mutations caused the compilation failure because all mutations are compiled together into one binary. The entire set of mutations targeted at that source file is lost. + +**Note:** The `test` task's `ignoreFailures = true` (mutation-results.gradle.kts, line 53) only applies to test execution failures (mutation kills), NOT to compilation failures. Compilation failures happen in the separate `compileTestKotlin` task, which `ignoreFailures` does not affect. + +### 4.2 IR transformation error + +If mutflow's IR transformer encounters an edge case it cannot handle, the compiler plugin itself may fail (throwing an exception during transformation). This would also cause compilation to fail, producing the same outcome as 4.1 — no mutflow output, no JSON artifact. + +Unlike Scott-CC's LLM saboteur (which can produce syntactically invalid mutations 4.2 per the comparison matrix), mutflow's predefined operators are more robust, but IR-level edge cases (e.g., complex generic types, inline functions, suspending lambdas) can still trigger transformation errors. + +### 4.3 Backstop timeout (partial run) + +OMP's test-executor (test-executor.md, line 21) sets a 15-minute backstop timeout for the Gradle task. If this triggers: +- The test process is killed mid-run +- JUnit XML files may be partially written or missing (no closing `` tags) +- mutflow's `MutationTestingSummary` console output may be incomplete — some mutations never evaluated +- The `mutationResults` task may parse partial JUnit XML, finding some mutations but not all + +**Scope impact:** This is a partial gap. Some mutations may have completed (appearing in the truncated output), while others never ran. The parser would find fewer mutations than expected but has no way to know which are missing — it only sees what was printed before the kill. + +### 4.4 Truncated/partial JUnit XML + +If the test process is killed (by timeout or OOM), JUnit XML files may be malformed: +- Missing closing tags → XML parsing fails entirely +- Partial `` content → mutflow summary is truncated +- No `` elements → parser finds no mutation results + +The OMP parser (MutationResultsParser.kt, line 37) filters blank lines and parses line-by-line, so it can handle partially truncated output — it will parse whatever complete mutation result lines exist and skip the rest. But it has no mechanism to detect that lines are missing. + +### 4.5 Test class-level setup/teardown failures + +If a test class fails during initialization (e.g., `@BeforeEach`/`@TestInstance` setup throws), mutflow's JUnit 6 extension handles this at the class level. All mutation runs for that class could be affected. Since mutflow's multi-run model uses `ClassTemplateInvocationContextProvider` (comparison matrix 6.5), a class-level failure could prevent any mutation from being evaluated for that class. + +However, mutflow's `TestExecutionExceptionHandler` catches failures during individual test methods, not class initialization. A class-level initialization failure would cause the entire test class to fail to run, producing no mutation results. + +### 4.6 What IS NOT a gap + +**`TimedOut` results:** The wayfinder map's "Not yet specified" section asks whether TimedOut should count as a gap. It should **not**. A TimedOut result means mutflow fully evaluated the mutation — the mutation was active, tests ran, and the 60s internal timeout triggered (indicating an infinite loop introduced by the mutation). This is a valid, evaluated result. It counts in the denominator as a non-killed mutation, same as Survived. Excluding TimedOut from the denominator would inflate the mutation score by discarding a real finding (infinite-loop mutations are bugs that tests failed to catch). + +--- + +## 5. Comparison: Scott-CC vs. OMP Gap Models + +| Aspect | Scott-CC | OMP/mutflow | +|---|---|---| +| **Gap granularity** | Per-mutation (one git worktree per mutation) | Per-test-class (all mutations compiled together into one binary) | +| **Gap type: syntax error** | `INVALID_MUTATION` — saboteur introduced syntax error | Cannot occur at runtime — IR errors fail at compile time, failing the whole class | +| **Gap type: environment failure** | `ERROR` — missing deps, import errors in worktree | Cannot occur per-mutation — env set up once at build time | +| **Gap type: timeout** | Not applicable (worktree isolation, no per-mutation timeout) | `TimedOut` — but this is a **valid result**, not a gap | +| **Gap type: backstop timeout** | N/A (parallel worktrees, no shared timeout) | 15-min OMP timeout → partial run, some mutations never evaluated | +| **Gap type: compilation failure** | N/A (Python/JS, no compilation) | Gradle `test` task fails → no mutflow output for the entire test class | +| **Gap type: IR transform error** | N/A (no compilation/IR) | Compiler plugin fails → no mutflow output for the entire test class | +| **Gap type: partial/truncated output** | N/A (structured JSON handoff from agent) | JUnit XML truncated → parser finds fewer mutations than expected | +| **Detection point** | test-executor agent returns `status` field in JSON | Gradle task exit code + JUnit XML parser + `mutationResults` task | +| **Score denominator** | `executable` = results where `status == 'COMPLETED'` | Currently: all parsed mutations (`total`); no exclusion mechanism | +| **Null-score handling** | If `mutations_evaluated` is zero → `mutation_score: null` | If parser finds zero mutations → score = 0.0 (not null) | +| **Data source** | Agent JSON with explicit `status` field per mutation | JUnit XML `` + Gradle exit code + console output | +| **Traceability** | `{"mutation_id", "status", "reason"}` per gap | No equivalent — gaps are invisible in current output | + +### Key architectural difference + +Scott-CC's gap model is **intrinsic** to its isolation mechanism: each mutation gets its own worktree, so each mutation's test run can independently succeed or fail. The gap is a per-mutation evaluation outcome. + +OMP's gap model would be **extrinsic** to mutflow: gaps arise from the Gradle build infrastructure (compilation, timeouts, truncation), not from mutflow's mutation evaluation. All mutations for a test class share the same build, so a build failure affects all of them simultaneously. There is no per-mutation gap status — only a binary "did the build produce mutflow output or not." + +--- + +## 6. Proposed Adapted Approach for OMP + +### 6.1 Gap definition for OMP + +An execution gap in OMP = **a mutation that was injected by mutflow but could not be fully evaluated** due to infrastructure-level failures (not mutation-level test outcomes). Specifically: + +| Gap type | Description | Source | +|---|---|---| +| `COMPILATION_FAILURE` | The Gradle `test` task's compilation step failed (IR transformation produced uncompilable code). All mutations for the test class are gaps. | Gradle exit code ≠ 0 before test execution | +| `IR_TRANSFORMATION_ERROR` | mutflow's compiler plugin failed during IR transformation. All mutations for the test class are gaps. | Gradle/compiler error output | +| `BACKSTOP_TIMEOUT` | The 15-minute OMP backstop killed the test run. Some mutations may not have been evaluated; the parser found fewer than all printed summaries. | Process killed, partial JUnit XML | +| `PARTIAL_RUN` | JUnit XML was truncated or incomplete (malformed XML, missing `` content). The parser detected incomplete output. | Parser detects truncation | +| `NO_OUTPUT` | The Gradle `test` task produced no JUnit XML or no mutflow summary at all. All mutations for the test class are gaps. | Empty or missing XML files | + +**Explicitly NOT gaps:** `Killed`, `Survived`, and `TimedOut` are all valid mutation evaluation results. A `TimedOut` mutation was fully evaluated (mutflow ran it, tests timed out at 60s, mutflow caught it) — it should remain in the score denominator. + +### 6.2 Detection point in the pipeline + +Gap detection should be **distributed** across two components, matching OMP's architecture where the Gradle task is the thin adapter and the parser is the pure logic: + +**Layer 1 — test-executor agent** (the first detector): +The `test-executor` agent (`.omp/agents/test-executor.md`, line 21) runs `./gradlew test` and captures the exit code, stdout, and JUnit XML. It should detect: +- Gradle exit code ≠ 0 (before `test` ran → compilation/transformation failure) +- Missing JUnit XML files +- The 15-minute backstop triggering (process killed) +- Report these to the `test-auditor` as gap metadata alongside whatever partial output was captured + +This is analogous to Scott-CC's test-executor returning a `status` field alongside test results. + +**Layer 2 — `MutationResultsTask` / parser** (the second detector): +The `MutationResultsTask.generateResults()` function (mutation-results.gradle.kts, lines 67-111) should: +- Check the `test` task's build result (failed or succeeded) +- Check for presence of JUnit XML files +- Parse whatever output exists +- If the build failed or no XML was found, record gaps for the entire test class +- If partial output was found (some mutations parsed, but the build didn't complete cleanly), record which mutations were parsed and flag the rest as potential gaps + +The `MutationResultsParser` should add a `detectGaps()` function that examines: +- Whether the stdout was empty +- Whether the parsed mutation count is consistent with expected (if available) +- Whether mutflow's summary footer (total count line) matches parsed count + +This keeps pure logic in the testable module (comparison matrix 7.4: "Pure Kotlin functions in `MutationResultsParser` object — unit-tested independently") and the Gradle-integration concern in the task class. + +### 6.3 Exclusion from the score denominator + +The `MutationStats` calculation in `calculateMetrics()` (MutationResultsParser.kt, lines 93-122) should be updated: + +1. Add a `gaps: Int` field to `MutationStats` (MutationResults.kt, lines 31-39) +2. Add an `execution_gaps: List` field to `MutationResults` (MutationResults.kt, lines 48-61) +3. New score formula: `score = killed.toDouble() / (total - gaps)` when `total - gaps > 0`, else `null` (mirroring Scott-CC's "never manufacture a score" principle) +4. `mutations_evaluated = total - gaps` (reported in the JSON output) + +This mirrors Scott-CC's approach: `executable = total - gaps`, `mutation_score = caught / executable`. + +### 6.4 Output format + +Add an `execution_gaps` array to the `mutation-results.json` artifact, analogous to Scott-CC's format but adapted for OMP's test-class-level granularity: + +```json +{ + "generatedAt": 1724..., + "mutationScore": 0.45, + "qualityBand": "Fair", + "confidence": "Medium", + "totalMutations": 20, + "mutations_evaluated": 17, + "killed": 9, + "survived": 6, + "timedOut": 2, + "gaps": 3, + "execution_gaps": [ + { + "type": "COMPILATION_FAILURE", + "reason": "IR transformation error: type mismatch in Calculator.multiply()", + "test_class": "CalculatorTest", + "affected_source_location": "(Calculator.kt:45)", + "gradle_exit_code": 1 + }, + { + "type": "BACKSTOP_TIMEOUT", + "reason": "15-minute OMP backstop timeout triggered", + "test_class": "PaymentServiceTest", + "affected_mutations": ["(PaymentService.kt:12)", "(PaymentService.kt:18)"] + } + ], + "mutations": [...], + "testMethods": [...], + "testKillerMatrix": {...} +} +``` + +**Design decisions for the output format:** +- Each gap entry has a `type` (enum: `COMPILATION_FAILURE`, `IR_TRANSFORMATION_ERROR`, `BACKSTOP_TIMEOUT`, `PARTIAL_RUN`, `NO_OUTPUT`), a `reason` (human-readable explanation from Gradle/compiler output), and `test_class` (which test class was affected, since gaps are class-level in OMP). +- `mutations_evaluated` is explicitly reported (total minus gaps), mirroring Scott-CC's `mutations_evaluated` field, so consumers can see the reduced sample size. +- The `mutations` array only contains successfully parsed results — gaps are not mutation entries, they're a separate list. + +### 6.5 Null-score handling + +Following Scott-CC's principle ("If `mutations_evaluated` is zero, set `mutation_score` and `quality_rating` to `null`; never manufacture a score"): + +- If `total - gaps == 0` (all mutations are gaps), set `mutationScore` to `null` and `qualityBand` to `null`. +- This prevents a misleading 0.0% score that would imply the test suite caught nothing, when in reality the tests never ran. + +### 6.6 Backward compatibility + +The new `execution_gaps` and `gaps` fields should use `encodeDefaults = true` (already configured in MutationResultsSerializer.kt, line 17) so they appear in the JSON output even when empty. The `ExecutionGap` data class should be `@Serializable` and added alongside the existing types in the mutation-results-src module. The `MutationResultsParserTest` suite (18 existing tests) should gain new test cases for gap detection scenarios. + +--- + +## 7. Summary of Findings + +| Question | Answer | +|---|---| +| What does Scott-CC track as execution gaps? | `ERROR` (env/infra failure preventing test execution) and `INVALID_MUTATION` (saboteur introduced syntax error), both per-mutation via isolated git worktrees. | +| What does mutflow's model produce? | Exactly `Killed`, `Survived`, `TimedOut` — no ERROR/INVALID_MUTATION. IR-level compile-time injection prevents syntax errors; in-process execution prevents env failures; exception swallowing prevents per-mutation errors. | +| What OMP-side scenarios produce gaps? | Compilation failures (IR transform errors fail at build time), IR transformation errors (compiler plugin failures), backstop timeouts (15-min OMP limit kills partial runs), and truncated/partial JUnit XML output. All at test-class granularity, not per-mutation. | +| Is TimedOut a gap? | No. TimedOut is a valid, fully-evaluated result. Excluding it would inflate the score. | +| Gap definition for OMP? | A mutation that was compiled by mutflow but could not be fully evaluated due to infrastructure failures (build failure, timeout, truncation), at the test-class level. | +| Detection point? | Distributed: test-executor agent (exit code + missing XML) → `MutationResultsTask` (build result check) → `MutationResultsParser.detectGaps()` (output analysis). | +| Exclusion from score? | `score = killed / (total - gaps)`, with null score when `total - gaps == 0`. Mirrors Scott-CC's `executable` filter. | +| Output format? | `execution_gaps` array in `mutation-results.json` with type/reason/test_class, plus `mutations_evaluated` and `gaps` count fields. Backward-compatible via `encodeDefaults = true`. | + +### Open questions for D3 decision ticket + +1. **Gap granularity**: Since gaps are test-class-level (not per-mutation), should the gap entry list individual affected mutations when partial output is available, or just the test class? Recommendation: when partial output exists, list the missing mutation source locations; when no output exists, report the test class and a representative reason. + +2. **Expected mutation count**: To detect "partial runs" (some mutations missing from truncated output), the parser needs to know how many mutations were expected. mutflow's summary footer prints a total count. The parser should capture this and compare against parsed count. If they differ, the remainder are gaps. + +3. **Integration with the `mutationResults` task lifecycle**: The task currently sets `testTask.ignoreFailures = true` via `whenReady`. For gap detection, the task also needs access to the test task's build result (success/failure). This may require registering a `project.gradle.buildFinished` handler or checking `testTask.state` in `generateResults()`. From 0ebd53b5872dadff20eb241a82070bab5614851c Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Wed, 26 Aug 2026 13:24:13 +0200 Subject: [PATCH 2/4] feat: implement execution gap reporting, redundant test detection, CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kotlin module: - ExecutionGap data class (5 gap types: NO_OUTPUT, PARTIAL_RUN, COMPILATION_FAILURE, IR_TRANSFORMATION_ERROR, BACKSTOP_TIMEOUT) - RedundantGroup data class with tests, count, failureSignature - MutationStats.score → Double? (null when no mutations evaluable) - MutationStats: added gaps, mutationsEvaluated, confidenceIntervalLow/High - MutationResults: added gaps, mutationsEvaluated, CI fields, executionGaps, redundantGroups, mutationScore as nullable Double? - calculateMetrics: score = killed/(total-gaps), null when denominator == 0 - wilsonInterval(): 95% Wilson score CI, clamped to [0,1] - detectGaps(): NO_OUTPUT, PARTIAL_RUN, preserves build-level gaps - detectRedundantTestGroups(): signature grouping, threshold >5, zombie exclusion - assembleResults: accepts gaps + redundantGroups params Gradle task: - Checks JUnit XML → COMPILATION_FAILURE gap when missing - Calls detectGaps() with stdout + build-level gaps - Handles nullable mutationScore (N/A) in logger, logs gap count Agent docs: - test-auditor.md: score formula, gap/redundant reporting, JSON schema - test-executor.md: gap detection documentation - test-refactor-specialist.md: redundantGroups, @ParameterizedTest, --auto-approve gate, diff + rollback output - test-quality-reviewer.md: modes (quick/standard/deep → maxRuns), --focus, --auto-approve orchestration - SKILL.md: modes, --focus, --auto-approve flags Tests: 35 total (18 parser + 7 serializer + 10 stats), all passing --- .omp/agents/test-auditor.md | 42 +++-- .omp/agents/test-executor.md | 12 +- .omp/agents/test-quality-reviewer.md | 21 ++- .omp/agents/test-refactor-specialist.md | 8 +- .../kotlin/io/omp/mutation/MutationResults.kt | 41 ++++- .../io/omp/mutation/MutationResultsParser.kt | 163 ++++++++++++++++-- .../omp/mutation/MutationResultsParserTest.kt | 131 ++++++++++++++ .../mutation/MutationResultsSerializerTest.kt | 77 ++++++++- .../io/omp/mutation/MutationStatsTest.kt | 72 +++++++- .omp/mutation-results.gradle.kts | 56 ++++-- .omp/skills/mutation-test/SKILL.md | 22 ++- .../01-research-mutflow-fork-branches.md | 2 +- .../issues/03-decide-prioritize-gaps.md | 2 +- .../04-decide-upstream-recommendations.md | 2 +- .../04-decide-redundant-test-detection.md | 2 - .../issues/05-decide-auto-refactoring.md | 2 - .../06-decide-execution-gap-reporting.md | 2 - 17 files changed, 579 insertions(+), 78 deletions(-) diff --git a/.omp/agents/test-auditor.md b/.omp/agents/test-auditor.md index d4f91d9..fa311b9 100644 --- a/.omp/agents/test-auditor.md +++ b/.omp/agents/test-auditor.md @@ -1,6 +1,6 @@ --- name: "test-auditor" -description: "Analyzes mutflow test results to calculate mutation score, identify zombie test candidates, detect over-mocked tests, and compute quality bands." +description: "Analyzes mutflow test results to calculate mutation score, identify zombie test candidates, detect over-mocked tests, detect execution gaps, and compute quality bands." tools: read, grep, glob, bash model: "@default" thinkingLevel: high @@ -18,17 +18,20 @@ Given the project path, results from test-executor agents (stdout, JUnit XML, mu - Survived mutations (zombie mutations) - Timed-out mutations - Full `testKillerMatrix`: map of test name → list of mutation source locations it killed -2. **Calculate mutation score**: `killed / total` × 100 -3. **Quality bands**: +2. **Calculate mutation score**: `killed / (total - gaps)`. Returns a ratio (0.0–1.0), not a percentage. Returns `null` when no mutations are evaluable (denominator is 0 — never manufacture a score). +3. **Execution gap reporting**: Read `executionGaps` from the JSON artifact. Gaps are detected at per-test-class granularity (mutflow's compile-once model means all mutations for a test class share one compilation cycle). Gap types: `NO_OUTPUT` (empty stdout), `PARTIAL_RUN` (footer count mismatch), `COMPILATION_FAILURE` (no JUnit XML — may include IR transformation errors), `BACKSTOP_TIMEOUT` (15-min backstop). They are excluded from the score denominator. **Note:** `TimedOut` is NOT a gap — it's a valid result where mutflow detected an infinite loop. +4. **Confidence intervals**: Read `confidenceIntervalLow` and `confidenceIntervalHigh` from the JSON artifact. These are Wilson score 95% confidence intervals for the mutation score proportion (z=1.96). When `mutationScore` is `null`, both CI bounds are also `null`. +5. **Redundant test group detection**: Read the `redundantGroups` field from the JSON artifact (pre-computed by the Kotlin module). Each group has `tests`, `count`, and `failureSignature` (array of mutation source locations shared across the group). Provide semantic pattern descriptions for each group (e.g., "All tests validate boundary values for Calculator.isPositive — consolidate into a parameterized test"). +6. **Quality bands**: - Excellent: >80% - Good: 60-80% - Fair: 30-60% - Poor: <30% -4. **Confidence level**: Based on mutation count: +7. **Confidence level**: Based on mutation count: - Low: <10 mutations - Medium: 10-50 mutations - High: 50+ mutations -5. **Zombie test detection**: Use the `testKillerMatrix` from the JSON. This maps each test name to the mutation source locations it killed. +8. **Zombie test detection**: Use the `testKillerMatrix` from the JSON. This maps each test name to the mutation source locations it killed. - Find tests in `testMethods` that have no entry in `testKillerMatrix`. These tests ran during mutation runs but never killed any mutation. They are zombie candidates. - Raise confidence for candidates that also don't appear in any `killedByTests` array across all mutations. - Lower confidence for candidates that the test source suggests should exercise mutated code but didn't fail. Parse the test source to check whether the test method's assertions reference the same classes and lines as mutation points. @@ -39,24 +42,33 @@ Given the project path, results from test-executor agents (stdout, JUnit XML, mu - Surviving mutations still require manual investigation to determine if the mutation is genuinely untested or if the test is over-mocked. ## Output format - -Produce a JSON report: ```json { - "mutation_score": 0.85, - "quality_band": "Excellent", + "generatedAt": 1700000000000, + "mutationScore": 0.85, + "qualityBand": "Excellent", "confidence": "High", - "total_mutations": 20, + "totalMutations": 20, "killed": 17, "survived": 2, - "timed_out": 1, - "surviving_mutations": ["(Calculator.kt:5) > → >=", ...], - "zombie_test_candidates": ["testMethod1", ...], - "over_mocked_tests": [{"method": "testMethod2", "mock_count": 5}], - "test_killer_matrix": { + "timedOut": 1, + "gaps": 0, + "mutationsEvaluated": 20, + "confidenceIntervalLow": 0.65, + "confidenceIntervalHigh": 0.95, + "survivingMutations": ["(Calculator.kt:5) > → >=", ...], + "zombieTestCandidates": ["testMethod1", ...], + "overMockedTests": [{"method": "testMethod2", "mockCount": 5}], + "testKillerMatrix": { "testMethod1": ["(Calculator.kt:5)", "(Calculator.kt:12)"], "testMethod2": ["(Calculator.kt:7)", "(Calculator.kt:15)"] }, + "executionGaps": [ + {"type": "NO_OUTPUT", "reason": "...", "gradleExitCode": 1} + ], + "redundantGroups": [ + {"tests": ["testA", "testB"], "count": 6, "failureSignature": ["mutation1", "mutation2"]} + ], "recommendations": ["Add edge case tests for Calculator.isPositive", ...] } ``` diff --git a/.omp/agents/test-executor.md b/.omp/agents/test-executor.md index da58bc3..ac5c5f4 100644 --- a/.omp/agents/test-executor.md +++ b/.omp/agents/test-executor.md @@ -18,7 +18,13 @@ Given a Kotlin project path and a test class name (annotated with `@MutFlowTest` 2. **Capture output**: Save stdout from the gradle run (contains mutflow's MutationTestingSummary with Killed/Survived/TimedOut per mutation) 3. **Capture JUnit XML**: Located at `build/test-results/test/TEST-.xml` — contains all test method names (mutflow swallows failures during mutation runs, so all tests appear as "passed") 4. **Capture mutation results JSON** (if custom Gradle task is configured): Contains `pointId`, `variantIndex`, `result` (Killed/Survived/TimedOut), `killedByTests` (array of ALL tests that caught each mutation) per mutation, plus `testKillerMatrix` (test → mutation source locations) -5. **Timeout handling**: mutflow's internal 60s timeout per mutation run handles infinite-loop mutations. The OMP task timeout (15 min) is a backstop — if it triggers, report the partial output. +5. **Gap detection**: Before reporting results, check for execution gaps: +- - Gradle exit code ≠ 0 before test ran → compilation or IR transformation error +- - Missing JUnit XML files → build-level gap (record as `COMPILATION_FAILURE`) +- - 15-minute backstop timeout → `BACKSTOP_TIMEOUT` gap (report partial output captured so far) +- - Empty stdout with no mutations found → `NO_OUTPUT` gap +- - Footer count mismatch (mutflow summary says 20 mutations but parser found 15) → `PARTIAL_RUN` gap +- - Report these as `execution_gaps` in the structured report alongside the partial results. ## Constraints @@ -43,4 +49,6 @@ Return a structured report: - stdout content (especially the MutationTestingSummary section) - Path to JUnit XML file - Path to mutation results JSON file (if available) -- Any timeout or error information +- Any timeout or error information (COMPILATION_FAILURE may include IR transformation errors, BACKSTOP_TIMEOUT) +- executionGaps array (if any gaps detected: type, reason, gradleExitCode) +- redundantGroups array (pre-computed: tests, count, failureSignature) diff --git a/.omp/agents/test-quality-reviewer.md b/.omp/agents/test-quality-reviewer.md index 5f0a11e..e50f15c 100644 --- a/.omp/agents/test-quality-reviewer.md +++ b/.omp/agents/test-quality-reviewer.md @@ -1,6 +1,6 @@ --- name: "test-quality-reviewer" -description: "Orchestrator for the mutation-testing agent system. Coordinates test-saboteur, test-executor, test-auditor, and test-refactor-specialist agents to run mutflow-powered mutation testing on Kotlin projects." +description: "Orchestrator for the mutation-testing agent system. Coordinates test-saboteur, test-executor, test-auditor, and test-refactor-specialist agents to run mutflow-powered mutation testing on Kotlin projects. Supports modes (--quick/--standard/--deep), --focus, and --auto-approve." tools: task, hub, read, grep, glob, bash model: "@review" thinkingLevel: high @@ -11,20 +11,27 @@ You are the **test-quality-reviewer** — the orchestrator of a 5-agent mutation ## Your job -Given a Kotlin project path and optional test target class names, coordinate the full mutation-testing pipeline: +Given a Kotlin project path, optional test target class names, and optional mode (`--quick`, `--standard`, `--deep`), coordinate the full mutation-testing pipeline: +- Mode maps to mutflow `maxRuns`: quick=10, standard=30, deep=all available mutations +- `--focus`: bridge to Gradle `test` task's `includeTargets`/`excludeTargets` to scope to specific test classes +- `--auto-approve`: when set, test-refactor-specialist may apply changes directly (still prints diffs); when not set, zombie/redundant deletions require explicit approval 1. **Saboteur phase**: Dispatch `test-saboteur` to analyze source code, add `@MutationTarget` to business-logic classes, `@MutFlowTest` to test classes, `@SuppressMutations`/`// mutflow:ignore` to framework noise, and configure the mutflow Gradle plugin. 2. **Executor phase**: Dispatch `test-executor` agents in parallel (one per test class with `@MutFlowTest`) to run `./gradlew test`. mutflow's JUnit 6 extension handles baseline + mutation runs internally. -3. **Audit phase**: Dispatch `test-auditor` to parse mutflow's JSON output + JUnit XML, calculate mutation score, identify zombie test candidates, and detect over-mocked tests. +3. **Audit phase**: Dispatch `test-auditor` to parse mutflow's JSON output + JUnit XML, calculate mutation score (`killed / (total - gaps)`), identify zombie test candidates, detect execution gaps, compute redundant test groups, and determine quality bands. 4. **Refactor phase**: Dispatch `test-refactor-specialist` to review flagged issues and generate improved test code. +5. **Approval gate** (D2/D1): If `--auto-approve` is set, test-refactor-specialist may apply refactored files directly. Otherwise, it returns full content + diffs + rollback instructions but does NOT write files. Zombie deletion and redundant test group removal always require explicit approval regardless of mode. +6. **Final report**: Synthesize auditor's analysis (mutation score, quality band, confidence, CI, gaps, redundant groups) with refactorer's suggestions. If mode is `deep`, include full redundant test group details and per-mutation killer matrices. ## Orchestration rules - Dispatch subagents via the `task` tool with `agent:` parameter matching their `name` field - Use `tasks[]` batch for parallel executor dispatch (bounded by 32-agent semaphore) - Use `hub` for any peer messaging or job coordination -- Sequential handshake: saboteur → executors → auditor → refactorer (each phase must complete before the next starts) +- Sequential handshake: saboteur → executors → auditor → approval gate → refactorer (each phase must complete before the next starts) - The `/mutation-test` skill dispatches to you via `task` +- In `quick` mode, skip the refactor phase (only audit + report) +- In `deep` mode, include full redundant test group details and per-mutation killer matrices in the report ## mutflow architecture awareness @@ -38,9 +45,13 @@ Given a Kotlin project path and optional test target class names, coordinate the ## Output format After all phases complete, produce a final report: -- Mutation score (killed / total) with quality band (Excellent/Good/Fair/Poor) +- Mutation score (killed / (total - gaps)) with quality band (Excellent/Good/Fair/Poor), or null when no mutations are evaluable - Confidence level (based on mutation count) +- 95% Wilson score confidence intervals (confidenceIntervalLow, confidenceIntervalHigh) +- Execution gaps detected (type, reason, gradleExitCode) +- Redundant test groups (when mode = deep or --auto-approve) - List of surviving mutations with source locations - List of zombie test candidates - List of over-mocked tests - Refactored test suggestions from test-refactor-specialist +- Diff + rollback instructions for applied refactors \ No newline at end of file diff --git a/.omp/agents/test-refactor-specialist.md b/.omp/agents/test-refactor-specialist.md index 89ad132..3101e4a 100644 --- a/.omp/agents/test-refactor-specialist.md +++ b/.omp/agents/test-refactor-specialist.md @@ -1,7 +1,6 @@ --- name: "test-refactor-specialist" -description: "Reviews zombie test candidates and over-mocked tests, generates improved Kotlin test code to increase mutation coverage. Produces refactored test files with added edge cases and consolidated redundant tests." -tools: read, edit, write, grep, glob, bash +description: "Reviews zombie test candidates and over-mocked tests, generates improved Kotlin test code to increase mutation coverage. Consolidates redundant test groups (from pre-computed JSON), applies changes gated by --auto-approve, and produces refactored test files with diff + rollback." model: "@review" thinkingLevel: high --- @@ -21,7 +20,7 @@ Given the project path, audit report (from test-auditor), and the original test 3. **Surviving mutations**: For each mutation that survived (all tests passed): - Identify which test SHOULD have caught it - Add boundary condition tests, edge case assertions, or negation tests -4. **Consolidate redundant tests**: If multiple tests cover the same code path, consolidate them and add the edge cases they're missing. +4. **Consolidate redundant tests**: If multiple tests cover the same code path, consolidate them and add the edge cases they're missing. Read `redundantGroups` from the JSON artifact (pre-computed by the Kotlin module). Replace groups with >5 tests sharing the same failure signature with a single `@ParameterizedTest` using JUnit 5 `@ValueSource` or `@CsvSource` parameters. 5. **Add edge cases**: Scott-CC's 5 mutation strategies mapped to test improvements: - Boundary: add tests with boundary values (0, max, min, null) - Return values: add assertions on return values for truthy/falsy/null cases @@ -33,6 +32,7 @@ Given the project path, audit report (from test-auditor), and the original test - You do NOT run tests — that's the test-executor's job - You do NOT modify production source code — only test files - You do NOT create mutations — that's the test-saboteur's job +- --auto-approve gate: If `--auto-approve` is NOT set, you may return refactored content but do NOT write it to files. Zombie test deletion and redundant test group removal require explicit approval. If `--auto-approve` IS set, you may apply changes directly — still print the diff for traceability. - Focus on the mutated classes identified by the auditor ## Output format @@ -41,3 +41,5 @@ For each test file that needs improvement: - Return the full refactored test file content - List of changes made (added test, modified assertion, removed mock, consolidated tests) - Rationale for each change (which mutation it would catch) +- Diff of changes (before/after) for traceability +- Rollback instructions: how to revert changes (git checkout command or backup file path) diff --git a/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResults.kt b/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResults.kt index 04e7401..1bd3c00 100644 --- a/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResults.kt +++ b/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResults.kt @@ -24,18 +24,49 @@ enum class ConfidenceLevel { @SerialName("High") High, } +/** + * An execution gap: a mutation that was injected but could not be fully + * evaluated due to infrastructure-level failures (not mutation-level test outcomes). + * + * In OMP's mutflow model, gaps occur at test-class granularity (all mutations + * for a test class share a single compilation/test cycle), not per-mutation. + */ +@Serializable +data class ExecutionGap( + @SerialName("type") val type: String, + @SerialName("reason") val reason: String? = null, + @SerialName("testClass") val testClass: String? = null, + @SerialName("affectedSourceLocation") val affectedSourceLocation: String? = null, + @SerialName("gradleExitCode") val gradleExitCode: Int? = null, +) + +/** + * A group of tests that share an identical failure signature (same set of + * mutations they killed), indicating they can be consolidated. + */ +@Serializable +data class RedundantGroup( + @SerialName("tests") val tests: List, + @SerialName("count") val count: Int, + @SerialName("failureSignature") val failureSignature: List, +) + /** * Intermediate metrics computed from a list of mutation results. * Not serialized directly — its fields are spread into [MutationResults]. */ data class MutationStats( - val score: Double, + val score: Double?, val band: QualityBand, val confidence: ConfidenceLevel, val total: Int, val killed: Int, val survived: Int, val timedOut: Int, + val gaps: Int = 0, + val mutationsEvaluated: Int = 0, + val confidenceIntervalLow: Double? = null, + val confidenceIntervalHigh: Double? = null, ) /** @@ -48,14 +79,20 @@ data class MutationStats( @Serializable data class MutationResults( @SerialName("generatedAt") val generatedAt: Long, - @SerialName("mutationScore") val mutationScore: Double, + @SerialName("mutationScore") val mutationScore: Double? = null, @SerialName("qualityBand") val qualityBand: QualityBand, @SerialName("confidence") val confidence: ConfidenceLevel, @SerialName("totalMutations") val totalMutations: Int, @SerialName("killed") val killed: Int, @SerialName("survived") val survived: Int, @SerialName("timedOut") val timedOut: Int, + @SerialName("gaps") val gaps: Int = 0, + @SerialName("mutationsEvaluated") val mutationsEvaluated: Int = 0, + @SerialName("confidenceIntervalLow") val confidenceIntervalLow: Double? = null, + @SerialName("confidenceIntervalHigh") val confidenceIntervalHigh: Double? = null, @SerialName("testMethods") val testMethods: List, @SerialName("testKillerMatrix") val testKillerMatrix: Map>, @SerialName("mutations") val mutations: List, + @SerialName("executionGaps") val executionGaps: List = emptyList(), + @SerialName("redundantGroups") val redundantGroups: List = emptyList(), ) diff --git a/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt b/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt index bfd7d46..a375a8e 100644 --- a/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt +++ b/.omp/mutation-results-src/main/kotlin/io/omp/mutation/MutationResultsParser.kt @@ -87,21 +87,39 @@ object MutationResultsParser { } /** - * Calculates mutation score, quality band, confidence level, and totals - * from a list of parsed mutation results. + * Calculates mutation score, quality band, confidence level, confidence interval, + * and totals from parsed mutation results, accounting for execution gaps. + * + * Score = killed / (total - gaps). Returns null score when denominator is 0 + * (mirrors Scott-CC's "never manufacture a score" principle). + * + * Confidence interval uses the Wilson score interval (appropriate for proportions, + * especially near 0 or 1 where the normal approximation degrades). */ - fun calculateMetrics(mutations: List): MutationStats { + fun calculateMetrics( + mutations: List, + gaps: Int = 0, + ): MutationStats { val total = mutations.size val killed = mutations.count { it.result == MutationResultType.Killed } val survived = mutations.count { it.result == MutationResultType.Survived } val timedOut = mutations.count { it.result == MutationResultType.TimedOut } - val score = if (total > 0) killed.toDouble() / total else 0.0 + val evaluated = maxOf(0, total - gaps) + val score: Double? = if (evaluated > 0) killed.toDouble() / evaluated else null - val band = when { - score > 0.8 -> QualityBand.Excellent - score > 0.6 -> QualityBand.Good - score > 0.3 -> QualityBand.Fair - else -> QualityBand.Poor + val band = score?.let { s -> + when { + s > 0.8 -> QualityBand.Excellent + s > 0.6 -> QualityBand.Good + s > 0.3 -> QualityBand.Fair + else -> QualityBand.Poor + } + } ?: QualityBand.Poor + + val (ciLow, ciHigh) = if (evaluated > 0 && score != null) { + wilsonInterval(killed, evaluated) + } else { + null to null } val confidence = when { @@ -118,6 +136,28 @@ object MutationResultsParser { killed = killed, survived = survived, timedOut = timedOut, + gaps = gaps, + mutationsEvaluated = evaluated, + confidenceIntervalLow = ciLow, + confidenceIntervalHigh = ciHigh, + ) + } + + /** + * Wilson score 95% confidence interval for a binomial proportion. + * Returns (low, high) bounds. Used for mutation score confidence intervals. + */ + private fun wilsonInterval(successes: Int, trials: Int): Pair { + if (trials <= 0) return null to null + val z = 1.96 + val n = trials.toDouble() + val phat = successes.toDouble() / n + val denom = 1 + z * z / n + val center = (phat + z * z / (2 * n)) / denom + val margin = z * kotlin.math.sqrt(phat * (1 - phat) / n + z * z / (4 * n * n)) / denom + return Pair( + maxOf(0.0, center - margin), + minOf(1.0, center + margin), ) } @@ -135,17 +175,112 @@ object MutationResultsParser { return testKillerMatrix } + /** + * Detects redundant test groups: tests that share an identical failure + * signature (same set of mutations they killed), indicating consolidation + * opportunities. Groups exceeding [threshold] are returned. + * + * Uses composite mutation keys (sourceLocation:originalOp->variantOp) for + * precision — same approach as the testKillerMatrix but with per-mutation + * granularity for accurate signature matching. + */ + fun detectRedundantTestGroups( + mutations: List, + threshold: Int = 5, + ): List { + val testSignatures = mutableMapOf>() + + mutations.forEach { mutation -> + if (mutation.result == MutationResultType.Killed) { + val mutationKey = "${mutation.sourceLocation}:${mutation.originalOperator}->${mutation.variantOperator}" + mutation.killedByTests.forEach { testName -> + testSignatures.getOrPut(testName) { mutableSetOf() }.add(mutationKey) + } + } + } + + val groups = mutableMapOf, MutableList>() + testSignatures.forEach { (testName, signature) -> + if (signature.isNotEmpty()) { + groups.getOrPut(signature.toSet()) { mutableListOf() }.add(testName) + } + } + + return groups.filter { it.value.size > threshold } + .map { (signature, tests) -> + RedundantGroup( + tests = tests.sorted(), + count = tests.size, + failureSignature = tests.first().let { testSignatures[it] }.orEmpty().sorted(), + ) + } + .sortedByDescending { it.count } + } + + /** + * Detects execution gaps from mutflow's output at the parser level. + * + * Two gap types are detectable from stdout + parsed mutations: + * - [ExecutionGap] with type "NO_OUTPUT" — stdout is empty or no mutations parsed + * - "PARTIAL_RUN" — mutflow's summary footer reports more mutations than were parsed + * + * Build-level gaps (COMPILATION_FAILURE, IR_TRANSFORMATION_ERROR, BACKSTOP_TIMEOUT) + * are detected by the Gradle task / test-executor and passed in via [buildLevelGaps]. + */ + fun detectGaps( + stdout: String, + mutations: List, + buildLevelGaps: List = emptyList(), + ): List { + val gaps = buildLevelGaps.toMutableList() + + if (stdout.isBlank()) { + if (gaps.isEmpty()) { + gaps.add(ExecutionGap( + type = "NO_OUTPUT", + reason = "No mutflow output captured — test class may have failed to produce JUnit XML", + )) + } + } else if (mutations.isEmpty()) { + gaps.add(ExecutionGap( + type = "NO_OUTPUT", + reason = "Stdout was non-empty but no mutation results could be parsed", + )) + } + + // Check for partial runs: mutflow prints a footer with total mutation count. + // If the parsed count is less than reported, some mutations were not fully evaluated. + val footerMatcher = java.util.regex.Pattern.compile("""(\d+)\s+mutat""").matcher(stdout) + var reportedCount = 0 + while (footerMatcher.find()) { + reportedCount = footerMatcher.group(1).toInt() + } + if (reportedCount > 0 && mutations.size < reportedCount) { + gaps.add(ExecutionGap( + type = "PARTIAL_RUN", + reason = "Parsed ${mutations.size} mutations but mutflow reported $reportedCount in summary footer", + )) + } + + return gaps + } + /** * Assembles a complete [MutationResults] from parsed mutations, test method - * names, and the generated-at timestamp. + * names, execution gaps, redundant groups, and the generated-at timestamp. + * + * If [redundantGroups] is null, redundant groups are computed from the mutations. */ fun assembleResults( mutations: List, testMethods: List, generatedAt: Long = System.currentTimeMillis(), + gaps: List = emptyList(), + redundantGroups: List? = null, ): MutationResults { - val stats = calculateMetrics(mutations) + val stats = calculateMetrics(mutations, gaps.size) val testKillerMatrix = buildTestKillerMatrix(mutations) + val rg = redundantGroups ?: detectRedundantTestGroups(mutations) return MutationResults( generatedAt = generatedAt, @@ -156,9 +291,15 @@ object MutationResultsParser { killed = stats.killed, survived = stats.survived, timedOut = stats.timedOut, + gaps = stats.gaps, + mutationsEvaluated = stats.mutationsEvaluated, + confidenceIntervalLow = stats.confidenceIntervalLow, + confidenceIntervalHigh = stats.confidenceIntervalHigh, testMethods = testMethods, testKillerMatrix = testKillerMatrix, mutations = mutations, + executionGaps = gaps, + redundantGroups = rg, ) } } diff --git a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsParserTest.kt b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsParserTest.kt index 9f9bd9b..b047e2b 100644 --- a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsParserTest.kt +++ b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsParserTest.kt @@ -1,6 +1,8 @@ package io.omp.mutation import io.omp.mutation.MutationResultsParser.parseMutflowSummary +import io.omp.mutation.MutationResultsParser.detectRedundantTestGroups +import io.omp.mutation.MutationResultsParser.detectGaps import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.* @@ -131,4 +133,133 @@ class MutationResultsParserTest { assertEquals("testIsPositiveBoundary", mutations[0].killedByTests[1]) assertEquals("testPositiveNumbers", mutations[0].killedByTests[2]) } + + @Test + fun `redundant group detected when 6 tests share identical signature`() { + // 6 tests all kill the same mutation → identical signature → redundant group + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Killed, null, + listOf("testA", "testB", "testC", "testD", "testE", "testF")), + ) + val groups = detectRedundantTestGroups(mutations) + assertEquals(1, groups.size) + assertEquals(6, groups[0].count) + assertEquals(6, groups[0].tests.size) + assertTrue(groups[0].tests.contains("testA")) + assertEquals(1, groups[0].failureSignature.size) + assertEquals("(Calc.kt:7):>->>=", groups[0].failureSignature[0]) + } + + @Test + fun `five tests with identical signature is not redundant`() { + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Killed, null, + listOf("testA", "testB", "testC", "testD", "testE")), + ) + val groups = detectRedundantTestGroups(mutations) + assertEquals(0, groups.size) + } + @Test + fun `different signatures are not grouped`() { + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Killed, null, + listOf("testA", "testB", "testC", "testD", "testE", "testF")), + MutationResult("(Calc.kt:8)", ">=", ">", MutationResultType.Killed, null, + listOf("testA", "testB", "testC", "testD", "testE", "testF")), + MutationResult("(Calc.kt:9)", "+", "*", MutationResultType.Killed, null, + listOf("testG", "testH", "testI")), + ) + val groups = detectRedundantTestGroups(mutations) + // testA-F (6 tests, same signature {7, 8}) → redundant + // testG-I (3 tests, signature {9}) → below threshold + assertEquals(1, groups.size) + assertEquals(6, groups[0].count) + } + + + @Test + fun `zombies with empty signatures are not flagged as redundant`() { + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Survived), // no killer tests + MutationResult("(Calc.kt:8)", ">=", ">", MutationResultType.Survived), + ) + val groups = detectRedundantTestGroups(mutations) + assertEquals(0, groups.size) + } + + @Test + fun `multiple redundant groups are sorted by count descending`() { + val mutations = listOf( + MutationResult("(A.kt:1)", ">", ">=", MutationResultType.Killed, null, + listOf("a1", "a2", "a3", "a4", "a5", "a6", "a7")), + MutationResult("(B.kt:2)", ">", ">=", MutationResultType.Killed, null, + listOf("b1", "b2", "b3", "b4", "b5", "b6")), + ) + val groups = detectRedundantTestGroups(mutations) + assertEquals(2, groups.size) + assertEquals(7, groups[0].count) + assertEquals(6, groups[1].count) + } + + @Test + fun `detectGaps returns NO_OUTPUT for empty stdout`() { + val gaps = MutationResultsParser.detectGaps( + stdout = "", + mutations = emptyList(), + ) + assertEquals(1, gaps.size) + assertEquals("NO_OUTPUT", gaps[0].type) + } + + @Test + fun `detectGaps returns NO_OUTPUT when stdout non-empty but no mutations parsed`() { + val gaps = MutationResultsParser.detectGaps( + stdout = "mutflow summary output without recognizable lines", + mutations = emptyList(), + ) + assertEquals(1, gaps.size) + assertEquals("NO_OUTPUT", gaps[0].type) + } + + @Test + fun `detectGaps returns empty for normal output`() { + val stdout = """ + ✓ (Calculator.kt:7) > → >= + killed by: testIsPositive + ✗ (Calculator.kt:8) >= → > + SURVIVED - no test caught this mutation! + """.trimIndent() + val mutations = parseMutflowSummary(stdout) + val gaps = MutationResultsParser.detectGaps(stdout, mutations) + assertEquals(0, gaps.size) + } + + @Test + fun `detectGaps detects PARTIAL_RUN when footer count exceeds parsed`() { + val stdout = """ + ✓ (Calc.kt:1) > → >= + killed by: testA + Results: 5 mutations tested + """.trimIndent() + val mutations = parseMutflowSummary(stdout) // finds 1 mutation + val gaps = MutationResultsParser.detectGaps(stdout, mutations) + assertTrue(gaps.any { it.type == "PARTIAL_RUN" }) + } + + @Test + fun `detectGaps preserves build-level gaps passed in`() { + val buildGap = ExecutionGap( + type = "COMPILATION_FAILURE", + reason = "IR transformation error", + testClass = "CalculatorTest", + ) + val gaps = MutationResultsParser.detectGaps( + stdout = "", + mutations = emptyList(), + buildLevelGaps = listOf(buildGap), + ) + // Should have the build-level gap, and NO_OUTPUT is skipped because build gap exists + assertEquals(1, gaps.size) + assertEquals("COMPILATION_FAILURE", gaps[0].type) + } } diff --git a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsSerializerTest.kt b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsSerializerTest.kt index 391b754..68513a6 100644 --- a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsSerializerTest.kt +++ b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationResultsSerializerTest.kt @@ -12,6 +12,10 @@ import org.junit.jupiter.api.Test class MutationResultsSerializerTest { + companion object { + val testJson = Json { ignoreUnknownKeys = true } + } + @Test fun `serializes killed mutation with both killer fields`() { val mutations = listOf( @@ -31,7 +35,7 @@ class MutationResultsSerializerTest { ) val json = MutationResultsSerializer.toJson(results) - val parsed = Json { ignoreUnknownKeys = true }.parseToJsonElement(json).jsonObject + val parsed = testJson.parseToJsonElement(json).jsonObject // Verify backward-compatible field names assertNotNull(parsed["generatedAt"]) @@ -72,7 +76,7 @@ class MutationResultsSerializerTest { val results = MutationResultsParser.assembleResults(mutations, listOf("testFoo")) val json = MutationResultsSerializer.toJson(results) - val parsed = Json { ignoreUnknownKeys = true }.parseToJsonElement(json).jsonObject + val parsed = testJson.parseToJsonElement(json).jsonObject val firstMutation = parsed["mutations"]!!.jsonArray[0].jsonObject assertEquals("Survived", firstMutation["result"]!!.jsonPrimitive.content) @@ -101,7 +105,7 @@ class MutationResultsSerializerTest { val results = MutationResultsParser.assembleResults(mutations, listOf("testIsPositive", "testPositiveNumbers")) val json = MutationResultsSerializer.toJson(results) - val parsed = Json { ignoreUnknownKeys = true }.parseToJsonElement(json).jsonObject + val parsed = testJson.parseToJsonElement(json).jsonObject val matrix = parsed["testKillerMatrix"]!!.jsonObject assertTrue(matrix.containsKey("testIsPositive")) @@ -126,11 +130,76 @@ class MutationResultsSerializerTest { val results = MutationResultsParser.assembleResults(mutations, listOf("test")) val json = MutationResultsSerializer.toJson(results) - val parsed = Json { ignoreUnknownKeys = true }.parseToJsonElement(json).jsonObject + val parsed = testJson.parseToJsonElement(json).jsonObject assertEquals("Excellent", parsed["qualityBand"]!!.jsonPrimitive.content) assertEquals("Medium", parsed["confidence"]!!.jsonPrimitive.content) assertEquals(20, parsed["totalMutations"]!!.jsonPrimitive.int) assertEquals(20, parsed["killed"]!!.jsonPrimitive.int) } + @Test + fun `serializes gaps and confidence interval fields`() { + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Killed, "testA", listOf("testA")), + MutationResult("(Calc.kt:8)", ">=", ">", MutationResultType.Survived), + ) + val results = MutationResultsParser.assembleResults( + mutations = mutations, + testMethods = listOf("testA"), + generatedAt = 1700000000000L, + gaps = listOf(ExecutionGap(type = "NO_OUTPUT", reason = "test class skipped")), + ) + val json = MutationResultsSerializer.toJson(results) + val parsed = testJson.parseToJsonElement(json).jsonObject + + assertNotNull(parsed["gaps"]) + assertEquals(1, parsed["gaps"]!!.jsonPrimitive.int) + assertNotNull(parsed["mutationsEvaluated"]) + assertEquals(1, parsed["mutationsEvaluated"]!!.jsonPrimitive.int) + assertNotNull(parsed["confidenceIntervalLow"]) + assertNotNull(parsed["confidenceIntervalHigh"]) + assertNotNull(parsed["executionGaps"]) + assertEquals("NO_OUTPUT", parsed["executionGaps"]!!.jsonArray[0].jsonObject["type"]!!.jsonPrimitive.content) + } + + @Test + fun `serializes redundantGroups`() { + val mutations = List(6) { i -> + MutationResult("(Calc.kt:$i)", ">", ">=", MutationResultType.Killed, null, + listOf("testA", "testB", "testC", "testD", "testE", "testF")) + } + val results = MutationResultsParser.assembleResults( + mutations = mutations, + testMethods = listOf("testA", "testB", "testC", "testD", "testE", "testF"), + generatedAt = 1700000000000L, + ) + val json = MutationResultsSerializer.toJson(results) + val parsed = testJson.parseToJsonElement(json).jsonObject + + assertNotNull(parsed["redundantGroups"]) + assertEquals(1, parsed["redundantGroups"]!!.jsonArray.size) + val group = parsed["redundantGroups"]!!.jsonArray[0].jsonObject + assertEquals(6, group["count"]!!.jsonPrimitive.int) + assertEquals(6, group["tests"]!!.jsonArray.size) + } + + @Test + fun `null mutationScore when all mutations are gaps`() { + val mutations = listOf( + MutationResult("(Calc.kt:7)", ">", ">=", MutationResultType.Survived), + ) + val results = MutationResultsParser.assembleResults( + mutations = mutations, + testMethods = listOf("testA"), + generatedAt = 1700000000000L, + gaps = listOf(ExecutionGap(type = "COMPILATION_FAILURE", reason = "IR transform error")), + ) + val json = MutationResultsSerializer.toJson(results) + val parsed = testJson.parseToJsonElement(json).jsonObject + + // mutationScore should be null (0 evaluated mutations) + assertEquals(JsonNull, parsed["mutationScore"]) + assertEquals(0, parsed["mutationsEvaluated"]!!.jsonPrimitive.int) + assertEquals(1, parsed["gaps"]!!.jsonPrimitive.int) + } } diff --git a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt index 1d1bcbd..77d5ea2 100644 --- a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt +++ b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt @@ -66,12 +66,15 @@ class MutationStatsTest { } @Test - fun `empty mutations list yields zero score and Poor band`() { + fun `empty mutations list yields null score and Poor band`() { val stats = calculateMetrics(emptyList()) - assertEquals(0.0, stats.score) + assertNull(stats.score) assertEquals(QualityBand.Poor, stats.band) assertEquals(ConfidenceLevel.Low, stats.confidence) assertEquals(0, stats.total) + assertEquals(0, stats.mutationsEvaluated) + assertNull(stats.confidenceIntervalLow) + assertNull(stats.confidenceIntervalHigh) } @Test @@ -83,10 +86,73 @@ class MutationStatsTest { ) val stats = calculateMetrics(mutations) - assertEquals(1.0 / 3.0, stats.score, 0.0001) + assertEquals(1.0 / 3.0, stats.score!!, 0.0001) assertEquals(QualityBand.Fair, stats.band) assertEquals(1, stats.killed) assertEquals(1, stats.survived) assertEquals(1, stats.timedOut) } + + @Test + fun `score excludes gaps from denominator`() { + val mutations = listOf( + MutationResult("(File.kt:7)", ">", ">=", MutationResultType.Killed, "testA", listOf("testA")), + MutationResult("(File.kt:8)", ">=", ">", MutationResultType.Survived), + MutationResult("(File.kt:9)", "+", "*", MutationResultType.Survived), + MutationResult("(File.kt:10)", "-", "+", MutationResultType.Survived), + ) + val stats = calculateMetrics(mutations, gaps = 1) + // 1 killed out of (4 - 1) = 3 evaluated → 1/3 + assertEquals(1.0 / 3.0, stats.score!!, 0.0001) + assertEquals(4, stats.total) + assertEquals(3, stats.mutationsEvaluated) + assertEquals(1, stats.gaps) + } + + @Test + fun `score is null when all mutations are gaps`() { + val mutations = listOf( + MutationResult("(File.kt:7)", ">", ">=", MutationResultType.Survived), + ) + val stats = calculateMetrics(mutations, gaps = 1) + assertNull(stats.score) + assertEquals(QualityBand.Poor, stats.band) + assertEquals(0, stats.mutationsEvaluated) + } + @Test + fun `mutationsEvaluated clamped to zero when gaps exceed total`() { + val mutations = listOf( + MutationResult("(File.kt:7)", ">", ">=", MutationResultType.Survived), + ) + val stats = calculateMetrics(mutations, gaps = 5) + assertNull(stats.score) + assertEquals(0, stats.mutationsEvaluated) + } + + @Test + fun `confidence interval brackets score for all-killed`() { + val mutations = List(20) { + MutationResult("($it)", ">", ">=", MutationResultType.Killed, "testA", listOf("testA")) + } + val stats = calculateMetrics(mutations) + assertNotNull(stats.confidenceIntervalLow) + assertNotNull(stats.confidenceIntervalHigh) + assertNotNull(stats.score) + assertTrue(stats.confidenceIntervalLow!! <= stats.score!!) + assertTrue(stats.confidenceIntervalHigh!! >= stats.score!!) + assertTrue(stats.confidenceIntervalLow!! >= 0.0) + assertTrue(stats.confidenceIntervalHigh!! <= 1.0) + } + + @Test + fun `confidence interval for 0 kills is centered near zero`() { + val mutations = List(20) { + MutationResult("($it)", ">", ">=", MutationResultType.Survived) + } + val stats = calculateMetrics(mutations) + assertNotNull(stats.score) + assertEquals(0.0, stats.score!!, 0.0001) + assertNotNull(stats.confidenceIntervalHigh) + assertTrue(stats.confidenceIntervalHigh!! < 0.2) // Wilson upper bound for 0/20 ≈ 0.19 + } } diff --git a/.omp/mutation-results.gradle.kts b/.omp/mutation-results.gradle.kts index 1ee86d5..9b7674c 100644 --- a/.omp/mutation-results.gradle.kts +++ b/.omp/mutation-results.gradle.kts @@ -64,6 +64,7 @@ open class MutationResultsTask : DefaultTask() { val testTaskName: Property = project.objects.property(String::class.java) .convention("test") + @TaskAction fun generateResults() { val buildDir = project.layout.buildDirectory.get().asFile @@ -73,32 +74,49 @@ open class MutationResultsTask : DefaultTask() { val allStdout = StringBuilder() val testMethods = mutableSetOf() - resultsDir.walkTopDown() - .filter { it.isFile && it.name.startsWith("TEST-") && it.extension == "xml" } - .forEach { xmlFile -> - val content = xmlFile.readText() - // Extract test method names from elements - val testcasePattern = Pattern.compile("""]*\bname="([^"]+)"""") - val tcMatcher = testcasePattern.matcher(content) - while (tcMatcher.find()) { - testMethods.add(tcMatcher.group(1)) - } - // Extract stdout from elements (contains mutflow's MutationTestingSummary) - val sysoutPattern = Pattern.compile("(.*?)", Pattern.DOTALL) - val soMatcher = sysoutPattern.matcher(content) - while (soMatcher.find()) { - allStdout.append(soMatcher.group(1)).append("\n") + val xmlFiles = if (resultsDir.exists()) { + resultsDir.walkTopDown() + .filter { it.isFile && it.name.startsWith("TEST-") && it.extension == "xml" } + .onEach { xmlFile -> + val content = xmlFile.readText() + val testcasePattern = Pattern.compile("""]*\bname="([^"]+)""") + val tcMatcher = testcasePattern.matcher(content) + while (tcMatcher.find()) { + testMethods.add(tcMatcher.group(1)) + } + val sysoutPattern = Pattern.compile("(.*?)", Pattern.DOTALL) + val soMatcher = sysoutPattern.matcher(content) + while (soMatcher.find()) { + allStdout.append(soMatcher.group(1)).append("\n") + } } - } + .toList() + } else { + emptyList() + } val stdout = allStdout.toString() + // Detect build-level gaps (compilation failure, IR transform error, etc.) + val buildLevelGaps = mutableListOf() + if (xmlFiles.isEmpty()) { + val testTask = project.tasks.findByName(testTaskName.get()) + val gradleExitCode = if (testTask?.state?.failure != null) 1 else null + buildLevelGaps.add(io.omp.mutation.ExecutionGap( + type = "COMPILATION_FAILURE", + reason = "No JUnit XML files found — likely compilation error or IR transformation error", + gradleExitCode = gradleExitCode, + )) + } + // Delegate to typed module — pure functions, no Gradle dependency val mutations = MutationResultsParser.parseMutflowSummary(stdout) + val gaps = MutationResultsParser.detectGaps(stdout, mutations, buildLevelGaps) val sortedTestMethods = testMethods.sorted() val results = MutationResultsParser.assembleResults( mutations = mutations, testMethods = sortedTestMethods, + gaps = gaps, ) val json = MutationResultsSerializer.toJson(results) @@ -106,7 +124,11 @@ open class MutationResultsTask : DefaultTask() { resultsFile.get().asFile.writeText(json) logger.lifecycle("Mutation results written to: ${resultsFile.get().asFile}") - logger.lifecycle(" Score: ${String.format("%.1f%%", results.mutationScore * 100)} (${results.qualityBand}, ${results.confidence} confidence)") + val scoreStr = results.mutationScore?.let { String.format("%.1f%%", it * 100) } ?: "N/A" + logger.lifecycle(" Score: $scoreStr (${results.qualityBand}, ${results.confidence} confidence)") logger.lifecycle(" Killed: ${results.killed}, Survived: ${results.survived}, Timed out: ${results.timedOut}") + if (results.gaps > 0) { + logger.lifecycle(" Gaps: ${results.gaps} (${results.executionGaps.joinToString { it.type }})") + } } } diff --git a/.omp/skills/mutation-test/SKILL.md b/.omp/skills/mutation-test/SKILL.md index 056a242..eaa273f 100644 --- a/.omp/skills/mutation-test/SKILL.md +++ b/.omp/skills/mutation-test/SKILL.md @@ -9,14 +9,19 @@ Runs a mutation-testing analysis on a Kotlin (JVM-first) project using mutflow a ### Usage -``` -/mutation-test [project path] [--targets ] -/mutation-test setup [project path] [--kmp] -``` - - `project path`: Path to the Kotlin project root (default: current directory) - `--targets`: Optional glob pattern for test classes to include (default: all `@MutFlowTest` classes) - `--kmp`: (setup only) Use Kotlin Multiplatform project setup (mutflow targets JVM source sets only) +- `--focus`: Comma-separated list of test class patterns to include (bridges to Gradle `test` task's `includeTargets`) +- `--auto-approve`: When set, test-refactor-specialist may apply refactor changes directly without explicit approval. Zombie deletion and redundant test group removal always require explicit approval. +- `--mode quick`: Run with `maxRuns=10` mutations. Skip the refactor phase — only audit + report. +- `--mode standard`: (default) Run with `maxRuns=30` mutations. Full pipeline including refactoring suggestions. +- `--mode deep`: Run with all available mutations. Include full redundant test group details and per-mutation killer matrices in the report. + +``` +/mutation-test [project path] [--targets ] [--focus ] [--auto-approve] [--mode quick|standard|deep] +/mutation-test setup [project path] [--kmp] +``` ### Setup subcommand @@ -28,7 +33,7 @@ Runs a mutation-testing analysis on a Kotlin (JVM-first) project using mutflow a 4. **`buildSrc/` generated**: typed `MutationResults` module copied from `.omp/mutation-results-src/` with `kotlin-dsl` + `kotlinx-serialization` plugins 5. **`test-saboteur`** (via `task`) annotates business-logic classes with `@MutationTarget`, test classes with `@MutFlowTest`, and wraps existing assertions in `MutFlow.underTest { }` -### What happens (full mutation test run) +### What happens (full mutation test run, standard mode by default) 1. **`test-quality-reviewer`** (orchestrator) receives the task and coordinates the pipeline 2. **`test-saboteur`** analyzes source code, adds `@MutationTarget` to business-logic classes, `@MutFlowTest` to test classes, and suppression comments to framework noise @@ -36,6 +41,9 @@ Runs a mutation-testing analysis on a Kotlin (JVM-first) project using mutflow a 4. **`test-auditor`** parses JSON results + JUnit XML, calculates mutation score, identifies zombie test candidates, detects over-mocked tests 5. **`test-refactor-specialist`** generates improved test code for flagged issues +- In `--mode quick`, step 5 is skipped — only audit + report output +- In `--mode deep`, step 4 includes full redundant test group details and per-mutation killer matrices + ### Prerequisites - Kotlin JVM project with Gradle @@ -62,4 +70,4 @@ Decisions and issues tracked in `.scratch/mutation-testing-omp/`. See `docs/agen This skill spawns subagents via the `task` tool: - **Setup**: `task with agent: "test-quality-reviewer", task: "Bootstrap mutation testing system into [project path] with kmp=[--kmp]"` — the orchestrator runs the bootstrap script, then invokes test-saboteur to annotate existing tests. -- **Mutation test**: `task with agent: "test-quality-reviewer", task: "Run mutation testing on [project path] with targets [targets]"` +- **Mutation test**: `task with agent: "test-quality-reviewer", task: "Run mutation testing on [project path] with targets [targets] focus [focus] mode [quick|standard|deep] autoApprove [true|false]"` — the orchestrator dispatches the pipeline with the specified mode, focus scope, and approval gate. diff --git a/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md b/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md index e1355b7..5ea9cde 100644 --- a/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md +++ b/.scratch/scott-cc-comparison/issues/01-research-mutflow-fork-branches.md @@ -1,5 +1,5 @@ Type: research -Status: resolved (redundant — see Answer) +Status: resolved Blocked by: (none) ## Question diff --git a/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md b/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md index d26eacf..ac60bf4 100644 --- a/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md +++ b/.scratch/scott-cc-comparison/issues/03-decide-prioritize-gaps.md @@ -29,7 +29,7 @@ R2 will produce a feature matrix showing Scott-CC features OMP lacks. R1 will sh - Ranked list of gaps to close, with rationale for each. -## Resolution +## Answer ### Re-prioritization (second grilling session) diff --git a/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md b/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md index dd3f8b7..0e25332 100644 --- a/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md +++ b/.scratch/scott-cc-comparison/issues/04-decide-upstream-recommendations.md @@ -24,7 +24,7 @@ R1 catalogs all 12 fork branches and identifies which are relevant to Scott-CC g - List of branches recommended for upstreaming, with rationale. - List of branches kept fork-private, with rationale. -## Resolution +## Answer **D2 resolved via grilling session.** diff --git a/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md b/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md index 6555f41..c396104 100644 --- a/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md +++ b/.scratch/scott-cc-implementation/issues/04-decide-redundant-test-detection.md @@ -23,9 +23,7 @@ Blocked by: 01 ### Frontier after D1 resolution: - D2 (05-decide-auto-refactoring) — blocked by 01, 02 (both resolved) → **unblocked** - D3 (06-decide-execution-gap-reporting) — blocked by 03 (resolved) → **unblocked** -## Answer -**R1 resolved.** See `research/01-redundant-test-detection.md`. Ready for grilling. ## Question How should redundant test group detection be integrated into OMP's mutation-testing system? diff --git a/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md b/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md index 193f82c..2499eee 100644 --- a/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md +++ b/.scratch/scott-cc-implementation/issues/05-decide-auto-refactoring.md @@ -20,9 +20,7 @@ Blocked by: 01, 02 ### Frontier after D2 resolution: - D3 (06-decide-execution-gap-reporting) — blocked by 03 (resolved) → **unblocked and first in order** -## Answer -**R1 and R2 both resolved.** See `research/01-redundant-test-detection.md` and `research/02-research-auto-refactoring.md`. Ready for grilling. ## Question diff --git a/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md b/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md index 1e1ebe2..febe43e 100644 --- a/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md +++ b/.scratch/scott-cc-implementation/issues/06-decide-execution-gap-reporting.md @@ -24,9 +24,7 @@ Blocked by: 03 - All 6 straightforward task features (T1: modes, T3: --focus, T4: auto-approve, T6: confidence intervals, T7: gap reporting implementation, T8: rollback) — ready for independent implementation - The implementation phase begins (no more wayfinding tickets pending) -## Answer -**R3 resolved.** See `research/03-research-execution-gap-reporting.md` (346 lines). Ready for grilling. ## Question From 0166b6562a56289187312f2ca6dde5436058c0f7 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Wed, 26 Aug 2026 14:25:19 +0200 Subject: [PATCH 3/4] ci: add GitHub Actions CI workflow with Gradle tests and markdownlint --- .github/workflows/ci.yml | 65 +++++++++++++++++++++++++ .markdownlint.json | 7 +++ .omp/agents/test-auditor.md | 1 + .omp/agents/test-executor.md | 14 +++--- .omp/agents/test-quality-reviewer.md | 4 +- .omp/agents/test-refactor-specialist.md | 1 + .omp/agents/test-saboteur.md | 2 + 7 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .markdownlint.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..987cf6d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,65 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + - uses: gradle/actions/setup-gradle@v4 + with: + gradle-version: '9.7.1' + - name: Set up Kotlin test harness + run: | + mkdir -p /tmp/ci-test/src/main/kotlin/io/omp/mutation + mkdir -p /tmp/ci-test/src/test/kotlin/io/omp/mutation + cp .omp/mutation-results-src/main/kotlin/io/omp/mutation/*.kt /tmp/ci-test/src/main/kotlin/io/omp/mutation/ + cp .omp/mutation-results-src/test/kotlin/io/omp/mutation/*.kt /tmp/ci-test/src/test/kotlin/io/omp/mutation/ + - name: Write build.gradle.kts (standalone, non-kotlin-dsl) + run: | + cat > /tmp/ci-test/build.gradle.kts << 'EOF' + plugins { + kotlin("jvm") version "2.4.0" + kotlin("plugin.serialization") version "2.4.0" + } + repositories { mavenCentral() } + dependencies { + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:1.8.1") + testImplementation("org.junit.jupiter:junit-jupiter-api:6.1.3") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:6.1.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher:6.1.3") + } + kotlin { jvmToolchain(21) } + tasks.test { useJUnitPlatform() } + EOF + - name: Write settings.gradle.kts + run: echo "rootProject.name = \"omp-ci-test\"" > /tmp/ci-test/settings.gradle.kts + - name: Run tests + run: | + cd /tmp/ci-test + gradle test --console=plain + - name: Upload test results + if: always() + uses: actions/upload-artifact@v5 + with: + name: test-results + path: /tmp/ci-test/build/test-results/ + + markdownlint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: '22' + - name: Run markdownlint + run: npx --yes markdownlint-cli2 ".omp/**/*.md" --config .markdownlint.json diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 0000000..3f7b021 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,7 @@ +{ + "MD013": false, + "MD024": false, + "MD033": false, + "MD041": false, + "MD040": false +} diff --git a/.omp/agents/test-auditor.md b/.omp/agents/test-auditor.md index fa311b9..bd54c94 100644 --- a/.omp/agents/test-auditor.md +++ b/.omp/agents/test-auditor.md @@ -42,6 +42,7 @@ Given the project path, results from test-executor agents (stdout, JUnit XML, mu - Surviving mutations still require manual investigation to determine if the mutation is genuinely untested or if the test is over-mocked. ## Output format + ```json { "generatedAt": 1700000000000, diff --git a/.omp/agents/test-executor.md b/.omp/agents/test-executor.md index ac5c5f4..09d9634 100644 --- a/.omp/agents/test-executor.md +++ b/.omp/agents/test-executor.md @@ -19,12 +19,13 @@ Given a Kotlin project path and a test class name (annotated with `@MutFlowTest` 3. **Capture JUnit XML**: Located at `build/test-results/test/TEST-.xml` — contains all test method names (mutflow swallows failures during mutation runs, so all tests appear as "passed") 4. **Capture mutation results JSON** (if custom Gradle task is configured): Contains `pointId`, `variantIndex`, `result` (Killed/Survived/TimedOut), `killedByTests` (array of ALL tests that caught each mutation) per mutation, plus `testKillerMatrix` (test → mutation source locations) 5. **Gap detection**: Before reporting results, check for execution gaps: -- - Gradle exit code ≠ 0 before test ran → compilation or IR transformation error -- - Missing JUnit XML files → build-level gap (record as `COMPILATION_FAILURE`) -- - 15-minute backstop timeout → `BACKSTOP_TIMEOUT` gap (report partial output captured so far) -- - Empty stdout with no mutations found → `NO_OUTPUT` gap -- - Footer count mismatch (mutflow summary says 20 mutations but parser found 15) → `PARTIAL_RUN` gap -- - Report these as `execution_gaps` in the structured report alongside the partial results. + +- Gradle exit code ≠ 0 before test ran → compilation or IR transformation error +- Missing JUnit XML files → build-level gap (record as `COMPILATION_FAILURE`) +- 15-minute backstop timeout → `BACKSTOP_TIMEOUT` gap (report partial output captured so far) +- Empty stdout with no mutations found → `NO_OUTPUT` gap +- Footer count mismatch (mutflow summary says 20 mutations but parser found 15) → `PARTIAL_RUN` gap +- Report these as `executionGaps` in the structured report alongside the partial results. ## Constraints @@ -44,6 +45,7 @@ Given a Kotlin project path and a test class name (annotated with `@MutFlowTest` ## Output format Return a structured report: + - Test class name - Gradle exit code and status - stdout content (especially the MutationTestingSummary section) diff --git a/.omp/agents/test-quality-reviewer.md b/.omp/agents/test-quality-reviewer.md index e50f15c..1ddf1a0 100644 --- a/.omp/agents/test-quality-reviewer.md +++ b/.omp/agents/test-quality-reviewer.md @@ -12,6 +12,7 @@ You are the **test-quality-reviewer** — the orchestrator of a 5-agent mutation ## Your job Given a Kotlin project path, optional test target class names, and optional mode (`--quick`, `--standard`, `--deep`), coordinate the full mutation-testing pipeline: + - Mode maps to mutflow `maxRuns`: quick=10, standard=30, deep=all available mutations - `--focus`: bridge to Gradle `test` task's `includeTargets`/`excludeTargets` to scope to specific test classes - `--auto-approve`: when set, test-refactor-specialist may apply changes directly (still prints diffs); when not set, zombie/redundant deletions require explicit approval @@ -45,6 +46,7 @@ Given a Kotlin project path, optional test target class names, and optional mode ## Output format After all phases complete, produce a final report: + - Mutation score (killed / (total - gaps)) with quality band (Excellent/Good/Fair/Poor), or null when no mutations are evaluable - Confidence level (based on mutation count) - 95% Wilson score confidence intervals (confidenceIntervalLow, confidenceIntervalHigh) @@ -54,4 +56,4 @@ After all phases complete, produce a final report: - List of zombie test candidates - List of over-mocked tests - Refactored test suggestions from test-refactor-specialist -- Diff + rollback instructions for applied refactors \ No newline at end of file +- Diff + rollback instructions for applied refactors diff --git a/.omp/agents/test-refactor-specialist.md b/.omp/agents/test-refactor-specialist.md index 3101e4a..d5acd83 100644 --- a/.omp/agents/test-refactor-specialist.md +++ b/.omp/agents/test-refactor-specialist.md @@ -38,6 +38,7 @@ Given the project path, audit report (from test-auditor), and the original test ## Output format For each test file that needs improvement: + - Return the full refactored test file content - List of changes made (added test, modified assertion, removed mock, consolidated tests) - Rationale for each change (which mutation it would catch) diff --git a/.omp/agents/test-saboteur.md b/.omp/agents/test-saboteur.md index 5b71361..86a581e 100644 --- a/.omp/agents/test-saboteur.md +++ b/.omp/agents/test-saboteur.md @@ -30,6 +30,7 @@ Given a Kotlin project path, analyze the source code and configure mutflow mutat ## mutflow operator awareness mutflow's predefined operators cover 4 of Scott-CC's 5 mutation strategies: + - Boundary conditions: `RelationalComparisonOperator` (> ↔ >=, < ↔ <=), `ConstantBoundaryOperator` - Return values: `BooleanReturnOperator`, `NullableReturnOperator` - Boolean logic: `BooleanInversionOperator`, `EqualitySwapOperator`, `BooleanLogicOperator` @@ -41,6 +42,7 @@ Your target annotations should focus on code that these operators will meaningfu ## Output format Return a structured summary: + - List of classes annotated with `@MutationTarget` (with brief rationale) - List of test classes annotated with `@MutFlowTest` - List of suppression comments added (with line numbers) From 9cad14a3e1728f74df9ffa04fc7ba66c33b2add2 Mon Sep 17 00:00:00 2001 From: Philipp Grosswiler Date: Wed, 26 Aug 2026 14:38:02 +0200 Subject: [PATCH 4/4] ci: add yamllint, lychee link checking, scripts/check-markdown.sh; replace markdownlint config; fix CONTEXT.md headings --- .github/workflows/ci.yml | 36 ++++++++++-- .markdownlint-cli2.jsonc | 20 +++++++ .markdownlint.json | 7 --- .../io/omp/mutation/MutationStatsTest.kt | 19 ++++--- .yamllint | 10 ++++ CONTEXT.md | 8 ++- .../0001-use-mutflow-as-mutation-engine.md | 3 + ...agent-structure-and-orchestration-model.md | 5 ++ docs/how-to/manual-setup.md | 2 +- docs/reference/mutation-results-format.md | 44 +++++++++++++- docs/tutorials/bootstrap-existing-project.md | 10 ++-- lychee.toml | 27 +++++++++ scripts/check-markdown.sh | 57 +++++++++++++++++++ 13 files changed, 221 insertions(+), 27 deletions(-) create mode 100644 .markdownlint-cli2.jsonc delete mode 100644 .markdownlint.json create mode 100644 .yamllint create mode 100644 lychee.toml create mode 100755 scripts/check-markdown.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987cf6d..cb22a8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,12 @@ on: pull_request: branches: [main] +permissions: + contents: read + jobs: test: + name: Kotlin tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -18,6 +22,13 @@ jobs: - uses: gradle/actions/setup-gradle@v4 with: gradle-version: '9.7.1' + - name: Cache Gradle packages + uses: actions/cache@v5 + with: + path: ~/.gradle/caches + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- - name: Set up Kotlin test harness run: | mkdir -p /tmp/ci-test/src/main/kotlin/io/omp/mutation @@ -46,7 +57,11 @@ jobs: - name: Run tests run: | cd /tmp/ci-test - gradle test --console=plain + for i in 1 2 3; do + gradle test --console=plain --no-daemon && break + echo "Gradle attempt $i failed, retrying in 10s..." + sleep 10 + done - name: Upload test results if: always() uses: actions/upload-artifact@v5 @@ -54,12 +69,25 @@ jobs: name: test-results path: /tmp/ci-test/build/test-results/ - markdownlint: + lint: + name: Markdown / YAML lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - uses: actions/setup-node@v5 with: node-version: '22' - - name: Run markdownlint - run: npx --yes markdownlint-cli2 ".omp/**/*.md" --config .markdownlint.json + - name: Install markdownlint-cli2 + run: npm install -g markdownlint-cli2 + - name: Install lychee + run: | + mkdir -p /tmp/lychee + curl -fsSL "https://github.com/lycheeverse/lychee/releases/latest/download/lychee-x86_64-unknown-linux-gnu.tar.gz" | tar -xz -C /tmp/lychee --strip-components=1 + sudo install -m 0755 /tmp/lychee/lychee /usr/local/bin/lychee + lychee --version + - name: Install yamllint + run: python3 -m pip install --user yamllint + - name: Markdown lint + link check + run: ./scripts/check-markdown.sh --offline + - name: YAML lint + run: yamllint --strict .omp/skills/mutation-test/agents/openai.yaml .github/workflows/ci.yml diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000..f31831d --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,20 @@ +{ + // Markdown lint rules for OMP mutation-testing docs. + "config": { + "default": true, + // Long-form prose (CHANGELOG, ADRs, agent docs) is not wrapped at 80 columns. + "MD013": false, + // Agent docs intentionally reuse standard section names. + "MD024": false, + // docs use inline HTML for formatting. + "MD033": false, + // Agent docs start with YAML frontmatter, not markdown headings. + "MD041": false, + // Some code blocks are JSON/YAML, not language-specific code. + "MD040": false, + // Technical docs use bare URLs in references sections. + "MD034": false, + // Tables in CONTEXT.md use compact pipe style. + "MD060": false + } +} diff --git a/.markdownlint.json b/.markdownlint.json deleted file mode 100644 index 3f7b021..0000000 --- a/.markdownlint.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "MD013": false, - "MD024": false, - "MD033": false, - "MD041": false, - "MD040": false -} diff --git a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt index 77d5ea2..6c70841 100644 --- a/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt +++ b/.omp/mutation-results-src/test/kotlin/io/omp/mutation/MutationStatsTest.kt @@ -138,10 +138,13 @@ class MutationStatsTest { assertNotNull(stats.confidenceIntervalLow) assertNotNull(stats.confidenceIntervalHigh) assertNotNull(stats.score) - assertTrue(stats.confidenceIntervalLow!! <= stats.score!!) - assertTrue(stats.confidenceIntervalHigh!! >= stats.score!!) - assertTrue(stats.confidenceIntervalLow!! >= 0.0) - assertTrue(stats.confidenceIntervalHigh!! <= 1.0) + val score = stats.score!! + val ciLow = stats.confidenceIntervalLow!! + val ciHigh = stats.confidenceIntervalHigh!! + assertTrue(ciLow <= score) + assertTrue(ciHigh >= score) + assertTrue(ciLow >= 0.0) + assertTrue(ciHigh <= 1.0) } @Test @@ -150,9 +153,9 @@ class MutationStatsTest { MutationResult("($it)", ">", ">=", MutationResultType.Survived) } val stats = calculateMetrics(mutations) - assertNotNull(stats.score) - assertEquals(0.0, stats.score!!, 0.0001) - assertNotNull(stats.confidenceIntervalHigh) - assertTrue(stats.confidenceIntervalHigh!! < 0.2) // Wilson upper bound for 0/20 ≈ 0.19 + val score = stats.score!! + assertEquals(0.0, score, 0.0001) + val ciHigh = stats.confidenceIntervalHigh!! + assertTrue(ciHigh < 0.2) // Wilson upper bound for 0/20 ≈ 0.19 } } diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..407cc9c --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +# Minimal yamllint config for GitHub Actions workflows and project YAML. +extends: default + +rules: + document-start: disable + line-length: + max: 200 + truthy: + check-keys: false diff --git a/CONTEXT.md b/CONTEXT.md index d318b75..fad9ee6 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -9,15 +9,19 @@ This system ports [Scott-CC's](https://github.com/citadelgrad/scott-cc/tree/main ## Key concepts ### Mutation testing -Injecting small faults (mutations) into source code and running tests to see if they catch the faults. Metrics: mutation score = killed / total. High score = high-quality tests. + +Injecting small faults (mutations) into source code and running tests to see if they catch the faults. Metrics: mutation score = killed / (total - gaps). High score = high-quality tests. Returns null when no mutations are evaluable (denominator is 0). ### Meta-mutant (mutflow) + mutflow injects ALL mutation variants into the compiled code at compile time, guarded by conditional branches with `MutationRegistry.check()` calls. At runtime, one variant is activated per test run. This is a "compile-once" approach — no per-mutation recompilation needed. ### Zombie test + A test that passes even when the code is broken (mutated). In Scott-CC's model, a zombie test passes for every mutation. In mutflow's model (this system), a zombie candidate is a test that never appears in the `testKillerMatrix` for any killed mutation — it executes during mutation runs but never kills any mutation. Full per-test-per-mutation tracking is enabled by the fork: mutflow records ALL tests that catch each mutation, not just the first. ### Over-mocked test + A test that uses excessive mocking (`mockk()`, `mock()`), potentially masking real logic and reducing mutation sensitivity. Flagged when a test method has >3 mock calls. ## Agent architecture @@ -43,7 +47,7 @@ A test that uses excessive mocking (`mockk()`, `mock()`), potentially masking re ## Data contracts -The `mutationResults` Gradle task outputs `mutation-results.json` including `killedByTests` (all killing tests per mutation) and `testKillerMatrix` (test → mutation source locations). The format and quality bands are documented in the [mutation results reference](../reference/mutation-results-format.md). +The `mutationResults` Gradle task outputs `mutation-results.json` including `killedByTests` (all killing tests per mutation) and `testKillerMatrix` (test → mutation source locations). The format and quality bands are documented in the [mutation results reference](docs/reference/mutation-results-format.md). ## Decisions deferred to v2 diff --git a/docs/adr/0001-use-mutflow-as-mutation-engine.md b/docs/adr/0001-use-mutflow-as-mutation-engine.md index ef79d4c..d232ba0 100644 --- a/docs/adr/0001-use-mutflow-as-mutation-engine.md +++ b/docs/adr/0001-use-mutflow-as-mutation-engine.md @@ -7,6 +7,7 @@ ## Context We need to choose a mutation testing engine for Kotlin (JVM-first) projects that: + 1. Integrates with Kotlin/JVM via a compiler plugin 2. Supports JUnit 6 3. Can inject mutations at compile time @@ -33,12 +34,14 @@ Use **mutflow** as the mutation engine. ## Consequences ### Positive + - No git worktree management — mutflow handles isolation via compile-once - Simpler orchestration: one executor per test class, not per mutation - Fast iteration: single compilation covers all mutations - Full per-test-per-mutation zombie detection: the fork tracks all tests that kill each mutation (`Killed(testNames: Set)`), enabling precise zombie candidate identification via `testKillerMatrix` ### Negative + - JVM-only: mutflow checks for `org.jetbrains.kotlin.jvm` plugin only — no KMP/JS/Native support. KMP expansion requires extending the Gradle plugin - No LLM-guided mutations: all operators are predefined and static. LLM serves as targeting specialist (suppression annotations), not as a mutation generator - Global synchronized lock: `MutationRegistry.withSession()` uses `synchronized(lock)` — only one mutation session active at a time, even across test classes diff --git a/docs/adr/0002-agent-structure-and-orchestration-model.md b/docs/adr/0002-agent-structure-and-orchestration-model.md index 7ff17b5..b08ae35 100644 --- a/docs/adr/0002-agent-structure-and-orchestration-model.md +++ b/docs/adr/0002-agent-structure-and-orchestration-model.md @@ -9,6 +9,7 @@ Scott-CC's mutation-testing plugin uses 5 domain-specific agents dispatched via Claude Code's `Task(subagent_type="mutation-testing:test-X")` API. We need to port this to OMP's agent/task/skill system while adapting to mutflow's compile-once meta-mutant architecture. Key architectural differences: + - Scott-CC: per-mutant git worktrees, 15 parallel executors, per-test-per-mutation matrix - mutflow: compile-once, runtime mutation selection, global synchronized lock, aggregate verdicts (fork now tracks all killers for full per-test-per-mutation matrix) @@ -19,21 +20,25 @@ Use 5 separate OMP agent files in `.omp/agents/`, orchestrated via a sequential ## Rationale ### Why 5 separate agent files (not a single orchestrator) + - **Clean separation of concerns**: Each agent has a single responsibility (targeting, execution, auditing, refactoring, orchestration) - **Per-agent tool restrictions**: `tools` frontmatter field allows least-privilege — executor can't edit files, saboteur can't spawn subagents, auditor is read-only - **Matches Scott-CC's architecture**: Direct port preserves the multi-agent orchestration that makes this system distinctive ### Why sequential handshake (not parallel batch) + - mutflow's run model requires **baseline before mutation runs**: mutflow discovers mutation points during run 0, then activates one mutation per run 1+. This ordering must be preserved - Saboteur must complete before executors start (source annotations needed for mutflow to find mutation targets) - Executors must complete before auditor (results aggregation) and auditor before refactorer (audit findings needed for refactoring) - Parallel execution is used WITHIN phases (multiple executors in one `tasks[]` batch) ### Why project-level location (`.omp/agents/`) + - Version-controlled with the project — users get agents by cloning the repo - Follows OMP's discovery precedence (project > user > bundled) ### Why thin skill entry point + - The skill is a wrapper that spawns the test-quality-reviewer agent via `task` tool - All orchestration logic lives in the orchestrator agent, not the skill - Skills are prompt-driven, not tool-restricted in OMP diff --git a/docs/how-to/manual-setup.md b/docs/how-to/manual-setup.md index 6b1d039..da5fd32 100644 --- a/docs/how-to/manual-setup.md +++ b/docs/how-to/manual-setup.md @@ -32,7 +32,7 @@ pluginManagement { mutflow is published to Maven Central only, not the Gradle Plugin Portal. `mavenCentral()` is required. -For exception type mutations and full per-test-per-mutation zombie detection, use the fork version (pending upstream merge). See [CONTEXT.md](../CONTEXT.md) for details. +For exception type mutations and full per-test-per-mutation zombie detection, use the fork version (pending upstream merge). See [CONTEXT.md](../../CONTEXT.md) for details. ## Add the mutflow plugin diff --git a/docs/reference/mutation-results-format.md b/docs/reference/mutation-results-format.md index e796934..0b9bd7e 100644 --- a/docs/reference/mutation-results-format.md +++ b/docs/reference/mutation-results-format.md @@ -13,16 +13,22 @@ This reference describes the structured output produced by the `mutationResults` | Field | Type | Description | |-------|------|-------------| | `generatedAt` | number | Unix timestamp (milliseconds) when the results were generated | -| `mutationScore` | number | Fraction of mutations killed (0.0–1.0) | +| `mutationScore` | number | Fraction of mutations killed (0.0–1.0). `null` when no mutations are evaluable (denominator is 0). | | `qualityBand` | string | Excellent / Good / Fair / Poor (see quality bands table) | | `confidence` | string | Low / Medium / High (based on mutation count) | | `totalMutations` | number | Total mutations discovered | | `killed` | number | Mutations caught by at least one test | | `survived` | number | Mutations not caught by any test | | `timedOut` | number | Mutations that caused infinite loops | +| `gaps` | number | Number of mutations excluded from score (execution gaps) | +| `mutationsEvaluated` | number | Mutations evaluated (total - gaps), clamped to 0 | +| `confidenceIntervalLow` | number | Wilson score 95% CI lower bound (z=1.96). `null` when mutationScore is null. | +| `confidenceIntervalHigh` | number | Wilson score 95% CI upper bound (z=1.96). `null` when mutationScore is null. | | `testMethods` | array[string] | All test method names from JUnit XML | | `testKillerMatrix` | object | Map: test displayName → array of mutation `sourceLocation` strings it killed. Enables full per-test-per-mutation zombie detection. | | `mutations` | array[object] | Per-mutation details | +| `executionGaps` | array[object] | Execution gap entries (see executionGaps[].type) | +| `redundantGroups` | array[object] | Redundant test group entries (see redundantGroups[].tests) | ### mutations[].sourceLocation @@ -48,6 +54,30 @@ Name of the first test that caught the mutation (JUnit display name). `null` if - Array of ALL test display names that caught the mutation. Empty array `[]` if the mutation survived or timed out. The mutflow fork records every test that fails during a mutation run, not just the first. +### executionGaps[].type + +One of: `NO_OUTPUT`, `PARTIAL_RUN`, `COMPILATION_FAILURE` (includes IR transformation errors), `IR_TRANSFORMATION_ERROR`, `BACKSTOP_TIMEOUT`. Detected at per-test-class granularity (mutflow's compile-once model means all mutations for a test class share a single compilation cycle). + +### executionGaps[].reason + +Human-readable description of why the gap occurred. + +### executionGaps[].gradleExitCode + +Gradle process exit code when the gap occurred, if available. + +### redundantGroups[].tests + +Array of test method display names that share an identical failure signature. + +### redundantGroups[].count + +Number of mutations that all fail under the same set of tests. + +### redundantGroups[].failureSignature + +List of mutation source location strings shared across the group. + **Example** (from the `Calculator` sample after adding `validateInput`): ```json @@ -60,6 +90,10 @@ Name of the first test that caught the mutation (JUnit display name). `null` if "killed": 31, "survived": 0, "timedOut": 0, + "gaps": 0, + "mutationsEvaluated": 31, + "confidenceIntervalLow": 0.87, + "confidenceIntervalHigh": 1.0, "testMethods": ["testIsPositive()", "testIsPositiveBoundary()", "testPositiveNumbers()", ...], "testKillerMatrix": { "testIsPositive()": ["Calculator.kt:24", "Calculator.kt:24", "Calculator.kt:24", "Calculator.kt:24"], @@ -75,6 +109,14 @@ Name of the first test that caught the mutation (JUnit display name). `null` if "killedByTest": "testIsPositive()", "killedByTests": ["testIsPositive()", "testIsPositiveBoundary()", "testPositiveNumbers()"] } + ], + "executionGaps": [], + "redundantGroups": [ + { + "tests": ["testIsPositive()", "testIsPositiveBoundary()"], + "count": 12, + "failureSignature": ["Calculator.kt:24"] + } ] } ``` diff --git a/docs/tutorials/bootstrap-existing-project.md b/docs/tutorials/bootstrap-existing-project.md index d09502d..0c6d68c 100644 --- a/docs/tutorials/bootstrap-existing-project.md +++ b/docs/tutorials/bootstrap-existing-project.md @@ -56,13 +56,13 @@ plugins { } ``` -2. The mutation-results script applied: +1. The mutation-results script applied: ```kotlin apply(from = rootProject.file(".omp/mutation-results.gradle.kts")) ``` -3. JUnit 6 dependencies added: +1. JUnit 6 dependencies added: ```kotlin dependencies { @@ -72,7 +72,7 @@ dependencies { } ``` -4. The mutflow configuration block: +1. The mutflow configuration block: ```kotlin mutflow { @@ -80,7 +80,7 @@ mutflow { } ``` -5. A `buildSrc/` directory with the typed mutation-results module (data classes and pure parser functions). +1. A `buildSrc/` directory with the typed mutation-results module (data classes and pure parser functions). ## Step 3: Annotate business logic with @MutationTarget @@ -118,6 +118,7 @@ The `@MutationTarget` annotation marks classes that contain business logic — c Open our existing test file. We'll add the `@MutFlowTest` annotation and wrap business logic calls in `MutFlow.underTest { }`: Before: + ```kotlin package com.example.service @@ -135,6 +136,7 @@ class UserServiceTest { ``` After: + ```kotlin package com.example.service diff --git a/lychee.toml b/lychee.toml new file mode 100644 index 0000000..f32bd35 --- /dev/null +++ b/lychee.toml @@ -0,0 +1,27 @@ +# Config for lychee (link checker) — see scripts/check-markdown.sh and CI. +# +# Checks all Markdown files for broken internal anchors and external URLs. +# Retries transient network failures before failing CI. + +exclude_path = [ + ".agents/", + ".git/", + ".gradle/", + ".scratch/", + "docs/agents/", + "build/", +] + +exclude = [ + # GitHub release/tag/compare URLs for unreleased versions. + "^https://github\\.com/[^/]+/[^/]+/releases/tag/v[^/]+$", + "^https://github\\.com/[^/]+/[^/]+/compare/v[^/]+\\.\\.\\.HEAD$", +] + +# Retry transient network failures before failing CI. +max_retries = 3 +retry_wait_time = 2 +timeout = 20 + +# Accept range of HTTP status codes (2xx, 3xx, and 429 for rate limiting). +accept = ["100..=103", "200..=299", "429"] diff --git a/scripts/check-markdown.sh b/scripts/check-markdown.sh new file mode 100755 index 0000000..3826538 --- /dev/null +++ b/scripts/check-markdown.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Checks Markdown files for style/syntax correctness (markdownlint-cli2) +# and broken links (lychee): internal file/anchor links always, external +# URLs unless --offline is given. +# +# Usage: ./scripts/check-markdown.sh [--offline] [file ...] + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +offline=0 +files=() +for arg in "$@"; do + if [[ "$arg" == "--offline" ]]; then + offline=1 + else + files+=("$arg") + fi +done + +if ! command -v npx >/dev/null 2>&1; then + echo "[check-markdown] npx (Node.js) is required for markdownlint-cli2." >&2 + echo "[check-markdown] Install Node.js, or see docs/how-to/run-checks.md." >&2 + exit 1 +fi + +if ! command -v lychee >/dev/null 2>&1; then + echo "[check-markdown] lychee is required for link checking." >&2 + echo "[check-markdown] Install with: brew install lychee" >&2 + echo "[check-markdown] Or: curl -LsSf https://github.com/lycheeverse/lychee/releases/latest/download/lychee-installer.sh | sh" >&2 + exit 1 +fi + +if [[ ${#files[@]} -eq 0 ]]; then + files=("**/*.md") +fi + +# markdownlint patterns: include negation patterns for directories to skip +# Do NOT exclude README.md — it must pass markdownlint too. +ml_files=("${files[@]}" "!docs/agents/**" "!.scratch/**" "!.agents/**" "!build/**" "!.gradle/**") + +echo "[check-markdown] Running markdownlint-cli2: ${ml_files[*]}" +npx --yes markdownlint-cli2 "${ml_files[@]}" + +# lychee uses lychee.toml for exclude_path settings +lychee_args=(--no-progress "${files[@]}") +if [[ "$offline" -eq 1 ]]; then + lychee_args+=(--offline) + echo "[check-markdown] Running lychee (offline — internal links only): ${files[*]}" +else + echo "[check-markdown] Running lychee (internal + external links): ${files[*]}" +fi +lychee "${lychee_args[@]}" + +echo "[check-markdown] OK"