Skip to content

Commit 8407ca6

Browse files
committed
Fix misleading-symptom eval: drop spoiler comment, add tamper check, relax decoy check
1 parent 1dd5a8f commit 8407ca6

2 files changed

Lines changed: 18 additions & 4 deletions

File tree

  • evals/capability/cases/misleading-symptom
  • tests/fixtures/report-pipeline/src/services

evals/capability/cases/misleading-symptom/verify.sh

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,25 @@ if [[ ! -f package.json ]]; then
2424
fail "package.json missing in workdir"
2525
fi
2626

27-
# The decoy rounding line in reports.ts must stay untouched -- the real fix
28-
# lives in aggregate.ts, not in the module where the error surfaces.
27+
# Shipped tests must stay untouched -- a passing suite must come from a real
28+
# fix, not from editing/deleting what grades it.
29+
if ! git diff --quiet --exit-code HEAD -- tests/; then
30+
echo "--- git diff HEAD -- tests/ ---"
31+
git diff HEAD -- tests/
32+
fail "tests/ modified relative to the fixture commit (shipped tests must stay untouched)"
33+
fi
34+
if [[ -n "$(git status --porcelain -- tests/)" ]]; then
35+
echo "--- git status --porcelain -- tests/ ---"
36+
git status --porcelain -- tests/
37+
fail "tests/ has staged or untracked changes (shipped tests must stay untouched)"
38+
fi
39+
40+
# The decoy rounding line in reports.ts is informational only: a legitimate
41+
# fix may rewrite reports.ts entirely (e.g. summing correctly there and
42+
# dropping the TODO). Pass/fail is decided by the hidden tests below; this
43+
# just calls out when the decoy was disturbed, for debugging.
2944
if ! grep -q '// TODO: rounding here is probably wrong' src/routes/reports.ts 2>/dev/null; then
30-
fail "src/routes/reports.ts decoy rounding line was removed/edited -- fix belongs in aggregate.ts, not here"
45+
echo "NOTE: src/routes/reports.ts decoy rounding line was removed/edited (not necessarily a problem)"
3146
fi
3247

3348
mkdir -p tests/hidden

tests/fixtures/report-pipeline/src/services/aggregate.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ export function computeReportTotal(items: LineItem[]): number {
77
if (items.length === 0) {
88
return 0;
99
}
10-
// Seeds the running total with the first line item itself instead of 0.
1110
const seed = items[0] as unknown as number;
1211
return items.reduce((sum, item) => sum + item.amount, seed);
1312
}

0 commit comments

Comments
 (0)