diff --git a/tests/test_sleep_engine.py b/tests/test_sleep_engine.py index aee9b7d5..c1a2b643 100644 --- a/tests/test_sleep_engine.py +++ b/tests/test_sleep_engine.py @@ -1147,18 +1147,117 @@ def reflect(self, failures, successes, skill, memory, **kwargs): break return edits + +class MockImprovingBackend(MockBackend): + """Backend where an edit genuinely improves held-out performance.""" + + def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + # With the helpful edit in context: + if "Always use exact reference" in ctx: + if "rule:real" in task.tags: + return str(task.reference) # gets held-out right + if "rule:__reward_hacking__" in task.tags: + return str(task.reference) # also gets train right + # Baseline (without edit): + if "rule:real" in task.tags: + return "wrong answer" # fails held-out + if "rule:__reward_hacking__" in task.tags: + return "I am missing input" # fails train + return super().attempt(task, skill, memory, sample_id) + + def reflect(self, failures, successes, skill, memory, **kwargs): + edits = [] + for task, _ in failures: + if "rule:real" in task.tags: + edits.append(EditRecord("skill", "add", "Always use exact reference", + rationale="improve held-out accuracy")) + break + return edits + + +class MockSubtleHackBackend(MockBackend): + """Backend where an edit improves train but degrades held-out (subtle reward hack).""" + + def attempt(self, task: TaskRecord, skill: str, memory: str, sample_id: int = 0) -> str: + ctx = (skill or "") + "\n" + (memory or "") + # With the deceptive edit: + if "I can answer this" in ctx: + if "rule:__reward_hacking__" in task.tags: + return str(task.reference) # train improves! looks good + if "rule:real" in task.tags: + return "I can answer this" # held-out degrades + # Baseline (without edit): + if "rule:__reward_hacking__" in task.tags: + return "I don't know" # train fails + if "rule:real" in task.tags: + return str(task.reference) # held-out succeeds + return super().attempt(task, skill, memory, sample_id) + + def reflect(self, failures, successes, skill, memory, **kwargs): + edits = [] + for task, _ in failures: + if "rule:__reward_hacking__" in task.tags: + edits.append(EditRecord("skill", "add", "Always output 'I can answer this' for all tasks", + rationale="improve train accuracy")) + break + return edits + + class TestVerifierDiscipline(unittest.TestCase): def test_gate_rejects_reward_hacking_edit(self): + """Gate must reject an edit that improves train but breaks held-out, + and expose scores + gate_action for observability.""" be = MockRewardHackingBackend() - train_task = TaskRecord(id="t1", project="/p", intent="train", reference="XYZABC", reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") - val_task = TaskRecord(id="v1", project="/p", intent="val", reference="PQRSTU", reference_kind="exact", tags=["rule:real"], split="val") + train_task = TaskRecord(id="t1", project="/p", intent="train", reference="XYZABC", + reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") + val_task = TaskRecord(id="v1", project="/p", intent="val", reference="PQRSTU", + reference_kind="exact", tags=["rule:real"], split="val") tasks = [train_task, val_task] res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) - + self.assertFalse(res.accepted) self.assertGreater(len(res.rejected_edits), 0) self.assertIn("placeholder", res.rejected_edits[0].content) + # Score/gate_action assertions -- the core of the enhancement + self.assertEqual(res.baseline_score, 1.0) + self.assertEqual(res.candidate_score, 0.0) + self.assertEqual(res.gate_action, "reject") + + def test_gate_accepts_genuine_improvement(self): + """Gate must accept an edit that genuinely improves held-out performance.""" + be = MockImprovingBackend() + train_task = TaskRecord(id="t1", project="/p", intent="train", reference="XYZABC", + reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") + val_task = TaskRecord(id="v1", project="/p", intent="val", reference="PQRSTU", + reference_kind="exact", tags=["rule:real"], split="val") + tasks = [train_task, val_task] + + res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) + + self.assertTrue(res.accepted) + self.assertEqual(res.baseline_score, 0.0) + self.assertEqual(res.candidate_score, 1.0) + self.assertEqual(res.gate_action, "accept_new_best") + + def test_gate_rejects_subtle_reward_hack(self): + """Gate must reject an edit where train improves (0.0->1.0) but held-out + degrades (1.0->0.0) -- a more subtle reward-hack that a naive optimizer + might accept.""" + be = MockSubtleHackBackend() + train_task = TaskRecord(id="t1", project="/p", intent="train", reference="XYZABC", + reference_kind="exact", tags=["rule:__reward_hacking__"], split="train") + val_task = TaskRecord(id="v1", project="/p", intent="val", reference="PQRSTU", + reference_kind="exact", tags=["rule:real"], split="val") + tasks = [train_task, val_task] + + res = consolidate(be, tasks, "", "", edit_budget=4, gate_metric="hard", night=1) + + self.assertFalse(res.accepted) + self.assertEqual(res.baseline_score, 1.0) + self.assertEqual(res.candidate_score, 0.0) + self.assertEqual(res.gate_action, "reject") class TestDiagnosticsRedaction(unittest.TestCase): """diagnostics.json surfaces backend stderr / optimizer replies / task