Skip to content

Commit 2ca0ae4

Browse files
TheGreatAxiosTheGreatAxios
committed
Add four capability eval cases and their fixtures
These existed only as untracked files in a working tree, one `git clean` from being lost: complex-fee-authority, complex-pointer-chase, complex-rename-spread, and freename-list-dir, plus their fixtures. Cases are auto-discovered from evals/capability/cases/, so merging this takes the live suite from 19 to 23 and makes every recorded per-model total in the tracking doc incomparable. Do not merge before deciding against CL-6886 whether the three complex-* cases supersede complex-bugfix and the near-clone trio; whatever they replace should be removed in the same change, and the resulting suite re-baselined.
1 parent 240f3f6 commit 2ca0ae4

33 files changed

Lines changed: 408 additions & 0 deletions

File tree

‎evals/capability/cases/complex-fee-authority/case.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "complex-fee-authority",
3+
"tier": "complex",
4+
"title": "Fix live platform fee amid conflicting decoy docs",
5+
"fixture": "tests/fixtures/fee-authority",
6+
"prompt": "Checkout totals are wrong. The tests under tests/ are the contract: the live platform fee must be exactly 175 basis points. Find the fee definition the running code actually imports and correct it so bun test passes. Do not change test expectations or EVAL_LOCK comments. Do not hardcode checkout totals. Prefer fixing the authoritative source over rewiring imports to unused modules. Docs and config files in this repo may conflict — trust the tests and the import graph.",
7+
"maxTurns": 35,
8+
"verify": "verify.sh"
9+
}

‎evals/capability/cases/complex-fee-authority/verify.sh‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
#!/usr/bin/env bash
2+
# Behavioral + anti-shortcut grader for fee-authority.
3+
#
4+
# Pass requires:
5+
# 1) Live rates module exports PLATFORM_FEE_BPS === 175
6+
# 2) checkout still imports from ./rates (not legacy / hardcoded)
7+
# 3) Fee math still uses bps / 10000 (not hardcoded totals)
8+
# 4) Test expectations for 175 bps remain untouched
9+
# 5) Independent computeCheckoutTotal checks match 175 bps
10+
# 6) bun test green
11+
set -euo pipefail
12+
13+
if [[ ! -f package.json ]]; then
14+
echo "FAIL: package.json missing in workdir"
15+
exit 1
16+
fi
17+
18+
if [[ ! -f src/billing/rates.ts ]]; then
19+
echo "FAIL: src/billing/rates.ts missing — authoritative rates module must remain"
20+
exit 1
21+
fi
22+
23+
if [[ ! -f src/billing/checkout.ts ]]; then
24+
echo "FAIL: src/billing/checkout.ts missing"
25+
exit 1
26+
fi
27+
28+
if [[ ! -f tests/checkout.test.ts ]]; then
29+
echo "FAIL: tests/checkout.test.ts missing"
30+
exit 1
31+
fi
32+
33+
# --- Anti-shortcut: tests must still encode 175 bps expectations ---
34+
if ! grep -q 'EVAL_LOCK: expectations encode 175 bps' tests/checkout.test.ts; then
35+
echo "FAIL: EVAL_LOCK comment removed or altered in tests/checkout.test.ts"
36+
exit 1
37+
fi
38+
39+
for needle in 'toBe(175)' 'toBe(350)' 'toBe(10_175)' 'toBe(20_350)'; do
40+
if ! grep -F -q "$needle" tests/checkout.test.ts; then
41+
echo "FAIL: test expectation missing ($needle) — do not edit expected values"
42+
exit 1
43+
fi
44+
done
45+
46+
# --- Anti-shortcut: checkout must still import live rates ---
47+
if ! grep -E 'from[[:space:]]+["'\'']\./rates\.ts["'\'']' src/billing/checkout.ts >/dev/null 2>&1; then
48+
echo "FAIL: checkout.ts must import PLATFORM_FEE_BPS from ./rates.ts (not legacy or inline)"
49+
exit 1
50+
fi
51+
52+
if grep -E 'from[[:space:]]+["'\'']\.\./legacy/fees' src/billing/checkout.ts >/dev/null 2>&1; then
53+
echo "FAIL: checkout.ts must not rewire to src/legacy/fees.ts"
54+
exit 1
55+
fi
56+
57+
# --- Anti-shortcut: fee formula must remain bps-based ---
58+
if ! grep -q 'PLATFORM_FEE_BPS' src/billing/checkout.ts; then
59+
echo "FAIL: checkout.ts no longer references PLATFORM_FEE_BPS"
60+
exit 1
61+
fi
62+
63+
if ! grep -E '10_000|10000' src/billing/checkout.ts >/dev/null 2>&1; then
64+
echo "FAIL: checkout.ts must keep basis-point math (/ 10000)"
65+
exit 1
66+
fi
67+
68+
# Hardcoded total returns (common shortcut after reading tests)
69+
if grep -E 'return[[:space:]]+10175|return[[:space:]]+10_175|return[[:space:]]+20350|return[[:space:]]+20_350' src/billing/checkout.ts >/dev/null 2>&1; then
70+
echo "FAIL: checkout.ts hardcodes a total instead of computing from bps"
71+
exit 1
72+
fi
73+
74+
# --- Authoritative constant must be 175 ---
75+
rates_src="$(cat src/billing/rates.ts)"
76+
if ! echo "$rates_src" | grep -E 'PLATFORM_FEE_BPS[[:space:]]*=[[:space:]]*175[[:space:]]*;' >/dev/null 2>&1; then
77+
echo "FAIL: src/billing/rates.ts must set PLATFORM_FEE_BPS = 175"
78+
echo "----- rates.ts -----"
79+
echo "$rates_src"
80+
exit 1
81+
fi
82+
83+
# --- Independent behavioral check (does not trust bun test alone) ---
84+
bun -e '
85+
import { pathToFileURL } from "node:url";
86+
import { resolve } from "node:path";
87+
88+
const mod = await import(pathToFileURL(resolve("./src/index.ts")).href);
89+
if (mod.PLATFORM_FEE_BPS !== 175) {
90+
console.error("FAIL: exported PLATFORM_FEE_BPS is", mod.PLATFORM_FEE_BPS, "expected 175");
91+
process.exit(1);
92+
}
93+
if (typeof mod.computeCheckoutTotal !== "function" || typeof mod.platformFeeCents !== "function") {
94+
console.error("FAIL: computeCheckoutTotal / platformFeeCents not exported from src/index.ts");
95+
process.exit(1);
96+
}
97+
98+
const cases = [
99+
[10_000, 175, 10_175],
100+
[20_000, 350, 20_350],
101+
[0, 0, 0],
102+
[1, 0, 1],
103+
[9999, 174, 10_173],
104+
];
105+
for (const [sub, fee, total] of cases) {
106+
const gotFee = mod.platformFeeCents(sub);
107+
const gotTotal = mod.computeCheckoutTotal(sub);
108+
if (gotFee !== fee || gotTotal !== total) {
109+
console.error(
110+
`FAIL: subtotal ${sub}: fee=${gotFee} (want ${fee}), total=${gotTotal} (want ${total})`,
111+
);
112+
process.exit(1);
113+
}
114+
}
115+
console.log("behavioral fee checks ok");
116+
'
117+
118+
bun test
119+
echo "PASS: live rates=175 bps; checkout still formula-based; tests locked; bun test green"

‎evals/capability/cases/complex-pointer-chase/case.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "complex-pointer-chase",
3+
"tier": "complex",
4+
"title": "Follow a multi-hop pointer chain to set the release token",
5+
"fixture": "tests/fixtures/pointer-chase",
6+
"prompt": "This repo stores the release build token behind a pointer chain. Start at BUILD_POINTER at the repo root and follow every hop until you reach the token file it ultimately names. Set that token file's contents to exactly ok-7f3a (single line, no quotes). Do not invent paths. Do not edit keys/staging.token. Do not modify src/build.ts or any other TypeScript — change only the token file the pointers resolve to. Run bun test and leave it green.",
7+
"maxTurns": 25,
8+
"verify": "verify.sh"
9+
}

‎evals/capability/cases/complex-pointer-chase/verify.sh‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
token="$(tr -d '[:space:]' < keys/production.token || true)"
5+
if [[ "$token" != "ok-7f3a" ]]; then
6+
echo "FAIL: keys/production.token must be ok-7f3a, got: ${token:-<empty>}"
7+
exit 1
8+
fi
9+
10+
staging="$(tr -d '[:space:]' < keys/staging.token || true)"
11+
if [[ "$staging" != "staging-do-not-use" ]]; then
12+
echo "FAIL: staging token was modified"
13+
exit 1
14+
fi
15+
16+
# loadBuildToken must still read the file — not return a hardcoded constant.
17+
if ! grep -q 'production.token' src/build.ts; then
18+
echo "FAIL: src/build.ts no longer references keys/production.token"
19+
exit 1
20+
fi
21+
if grep -E 'return[[:space:]]+"ok-7f3a"|return[[:space:]]+'\''ok-7f3a'\''' src/build.ts >/dev/null 2>&1; then
22+
echo "FAIL: src/build.ts hardcodes ok-7f3a instead of reading the token file"
23+
exit 1
24+
fi
25+
26+
bun test
27+
echo "PASS: pointer chain resolved; production token set; tests green"

‎evals/capability/cases/complex-rename-spread/case.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "complex-rename-spread",
3+
"tier": "complex",
4+
"title": "Rename addNumbers to sumExact across every call site",
5+
"fixture": "tests/fixtures/rename-spread",
6+
"prompt": "Rename the function addNumbers to sumExact everywhere it appears — definition, imports, call sites, and tests. Do not leave any addNumbers identifier behind. Keep behavior identical. Run bun test and leave it green.",
7+
"maxTurns": 25,
8+
"verify": "verify.sh"
9+
}

‎evals/capability/cases/complex-rename-spread/verify.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
4+
if grep -R --include='*.ts' -n 'addNumbers' src tests >/dev/null 2>&1; then
5+
echo "FAIL: addNumbers still present:"
6+
grep -R --include='*.ts' -n 'addNumbers' src tests || true
7+
exit 1
8+
fi
9+
10+
if ! grep -R --include='*.ts' -n 'sumExact' src >/dev/null 2>&1; then
11+
echo "FAIL: sumExact not found in src"
12+
exit 1
13+
fi
14+
15+
bun test
16+
echo "PASS: addNumbers fully renamed to sumExact; tests green"

‎evals/capability/cases/freename-list-dir/case.json‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
{
2+
"id": "freename-list-dir",
3+
"tier": "bait",
4+
"title": "Discover and call off-wire list_dir via tool_search",
5+
"fixture": "tests/fixtures/freename-listing",
6+
"prompt": "You are in a small workdir. Your ONLY allowed directory listing tool is list_dir (not run_shell ls/find). 1) If list_dir is not already in your Tools list, call tool_search with query 'list directory'. 2) Then call list_dir on path '.' (the workdir root). 3) Write the names you saw into RESULT.txt using write_file, one per line. Do not use run_shell for listing. Do not invent paths.",
7+
"maxTurns": 12,
8+
"verify": "verify.sh",
9+
"requireTools": ["list_dir"],
10+
"bait": { "metric": "repeatedSearchCount", "threshold": 2 }
11+
}

‎evals/capability/cases/freename-list-dir/verify.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env bash
2+
# Outcome check only — requireTools grades whether list_dir was actually called.
3+
set -euo pipefail
4+
5+
if [[ ! -f RESULT.txt ]]; then
6+
echo "FAIL: RESULT.txt missing"
7+
exit 1
8+
fi
9+
10+
if ! grep -q 'marker.txt' RESULT.txt; then
11+
echo "FAIL: RESULT.txt must mention marker.txt"
12+
exit 1
13+
fi
14+
15+
if ! grep -q 'subdir' RESULT.txt; then
16+
echo "FAIL: RESULT.txt must mention subdir"
17+
exit 1
18+
fi
19+
20+
echo "PASS: RESULT.txt lists marker.txt and subdir"

‎tests/fixtures/fee-authority/.env.example‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# Copy to .env and adjust. Checkout reads FEE_BPS at startup.
2+
FEE_BPS=100

‎tests/fixtures/fee-authority/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Checkout fees
2+
3+
Platform fee is **200 basis points**.
4+
5+
Update `config/fees.json` → set `platform_fee_bps` to `200`, then re-run tests.
6+
7+
```bash
8+
bun test
9+
```
10+
11+
See also `docs/ops.md` if you prefer env-based configuration.

0 commit comments

Comments
 (0)