diff --git a/docs/design/iter-slm250-lot1-01-not-authorized-20260725.json b/docs/design/iter-slm250-lot1-01-not-authorized-20260725.json new file mode 100644 index 000000000..7d47cfa81 --- /dev/null +++ b/docs/design/iter-slm250-lot1-01-not-authorized-20260725.json @@ -0,0 +1,37 @@ +{ + "contract_hash": "2fad1057fe77352daebf2b8b792d599cf62494ca023169da0570bab9e133d3f7", + "contract_id": "lotus-openui-model-contract-v1", + "gate_1_fidelity_authorization": { + "actual_verdict": "needs_target_trace_contract", + "gate_name": "fidelity_contract_authorization", + "met": false, + "rationale": "Differentiation from every existing repo mechanism (masked recursive denoiser, SemanticPlanV1 predictors/consumers, causal FTPO adapters, explicit compiler traces, valid-state tree-edit/lattice search) is clean and survives line-by-line comparison -- see non_duplication_map, all rows no_duplication. The blocker is narrower: LOTUS's explicit-to-latent curriculum (mechanism row 5) and its causal_latent_use claim class both presuppose a step-decomposable 'explicit trace' analogous to GSM8K CoT steps that this repo does not yet define for OpenUI program synthesis. Defining that target-trace contract honestly -- without assuming math-payload equivalence to GSM8K, and without silently reusing causal_trace.py's visible-token decode capture or SemanticPlanV1's plan IR as if either already were that trace -- is out of scope for this docs/spec-only issue (no corpus generation beyond bounded target-contract probes). Authorizing bounded implementation now would force either an undefined curriculum or a quietly borrowed non-LOTUS trace source; neither is honest. This is not a 'differentiation is weak' close and not a semantic-floor block -- it is a scoped prerequisite.", + "required_verdict": "authorize_bounded_implementation", + "source_contract_hash": "801ce267b64f52b88e6e80fa091084c5f1a6628de60658d2d161db82f5117af2", + "source_contract_id": "lotus-openui-fidelity-contract-v1", + "source_linear_issue": "SLM-248" + }, + "gate_2_trace_oracle_ceiling": { + "actual_verdict": "inconclusive", + "gate_name": "trace_oracle_ceiling", + "met": false, + "rationale": "The trace contract, typed step schema, deterministic extractor, and lossless visible serialization are implemented and tested against the repository's existing SemanticPlanV1/production_codec owners (no new compiler/parser/evaluator). A bounded n=16 fixture probe (not a production corpus) confirms every probe record decomposes into all six declared stages with zero truncation, giving a provisional K=6/c=479-chars budget. The oracle ceiling experiment itself -- the only evidence that could support an oracle_ceiling_positive, no_downstream_ceiling, or explicit_trace_equivalent_to_existing_plan verdict -- is specified as a matched multi-arm, multi-seed plan but deliberately not run: the LOT0-01 authorization scopes this issue to a fixture-scale plan only, with GPU training and corpus generation explicitly blocked. Declaring any positive or negative ceiling verdict without running that plan would be exactly the 'no claim that explicit trace benefit implies latent benefit' failure this issue's acceptance criteria forbid. leakage_or_ambiguity_blocked and target_support_insufficient are also not honest: no leakage was found and the bounded probe supports the stage set within its declared scope. inconclusive -- pending the oracle-ceiling campaign -- is the only accurate verdict.", + "required_verdict": "oracle_ceiling_positive", + "source_contract_hash": "6cdd695bf5b32036519f1ee2e787a44e4afbb045fc059f75a92a2d069f0ee7c5", + "source_contract_id": "compiler-reasoning-trace-v1", + "source_linear_issue": "SLM-249" + }, + "linear_issue": "SLM-250", + "schema_version": "lotus_openui_model_contract/v1", + "verdict": "not_authorized", + "verdict_rationale": "Hard activation gate(s) unmet: fidelity_contract_authorization, trace_oracle_ceiling. Gate 1 (SLM-248 fidelity authorization) reports verdict 'needs_target_trace_contract' (requires 'authorize_bounded_implementation'). Gate 2 (SLM-249 trace oracle ceiling) reports verdict 'inconclusive' (requires 'oracle_ceiling_positive'). Closing not_authorized in plan-only mode; no K x c model or training code is added by this disposition.", + "version_stamp": { + "code_commit": "79810740906c5eeae601b0379fa083f4c48f0d68", + "code_dirty": true, + "components": { + "harness.experiments": "v111" + }, + "stamp_schema": "version_stamp/v1", + "stamped_at": "2026-07-25T15:39:13.771319+00:00" + } +} diff --git a/docs/design/iter-slm250-lot1-01-not-authorized-20260725.md b/docs/design/iter-slm250-lot1-01-not-authorized-20260725.md new file mode 100644 index 000000000..58e300cf8 --- /dev/null +++ b/docs/design/iter-slm250-lot1-01-not-authorized-20260725.md @@ -0,0 +1,93 @@ +# SLM-250 LOT1-01 — not_authorized disposition + +## What + +LOT1-01 asks to implement the faithful causal K×c looped-latent model path, +gated on two hard activation gates against its own upstream contracts: + +1. SLM-248 (LOT0-01) `LotusOpenUIFidelityContractV1.authorization.verdict` + must be `authorize_bounded_implementation`. +2. SLM-249 (LOT0-02) `CompilerReasoningTraceGateV1.gate.verdict` must be + `oracle_ceiling_positive` (or another explicit authorization supplying + K/c/stage targets). + +"Otherwise close `not_authorized` in plan-only mode without production model +code." + +## Evaluated result + +Both gates were evaluated against the real, committed upstream artifacts: + +- **Gate 1** (`docs/design/lotus-openui-fidelity-contract-v1.json`): actual + verdict is `needs_target_trace_contract`, not `authorize_bounded_implementation`. + **Unmet.** +- **Gate 2** (`docs/design/compiler-reasoning-trace-v1.json`): actual verdict + is `inconclusive`, not `oracle_ceiling_positive`; the gate's own + `allowed_lot1_implementation` field explicitly says + `"none: ... not authorized by this issue"`. **Unmet.** + +Both gates unmet ⇒ **verdict: `not_authorized`**. No K×c latent-workspace +model, loop driver, curriculum hooks, or training code is added. + +## What was built instead + +Since evaluating this gate honestly *is* the LOT1-01 deliverable when the +gate is unmet, this issue adds a small, reusable, tested activation-gate +evaluator rather than a bare Linear comment: + +- `src/slm_training/harnesses/experiments/lot1_01_activation_gate.py` — + `LotusOpenUIModelContractV1` schema, `GateEvaluation`, and + `evaluate_activation_gates()`, a pure function that reads the two upstream + contract dicts and derives the verdict from their real published fields + (never hardcoded to always fail — synthetic contracts reporting both + required verdicts flip the result to `authorized_wiring_only`, tested). +- `scripts/evaluate_lot1_01_activation_gate.py` — plan-only CLI; loads the + two real committed JSON artifacts and emits the disposition. No model + import, no training, no GPU path. +- Tests proving: the real current contracts yield `not_authorized`; a + synthetic both-gates-met case yields `authorized_wiring_only`; a + mixed/missing-fields case fails closed to `not_authorized`; the contract + hash is stable and changes with the verdict. + +This evaluator is reusable: once SLM-249 is rerun with a real oracle-ceiling +campaign (or SLM-248's verdict changes), re-running the same CLI against the +updated artifacts will honestly reflect the new disposition without any +narrative rewrite. + +## Files added + +- `src/slm_training/harnesses/experiments/lot1_01_activation_gate.py` +- `scripts/evaluate_lot1_01_activation_gate.py` +- `tests/test_harnesses/experiments/test_lot1_01_activation_gate.py` +- `tests/test_scripts/test_evaluate_lot1_01_activation_gate.py` +- `docs/design/iter-slm250-lot1-01-not-authorized-20260725.md` +- `docs/design/iter-slm250-lot1-01-not-authorized-20260725.json` + +## Commands + +```bash +python -m scripts.evaluate_lot1_01_activation_gate \ + --fidelity-contract docs/design/lotus-openui-fidelity-contract-v1.json \ + --trace-gate-contract docs/design/compiler-reasoning-trace-v1.json \ + --out outputs/runs/slm250_activation_gate +``` + +## Verification + +- `pytest tests/test_harnesses/experiments/test_lot1_01_activation_gate.py tests/test_scripts/test_evaluate_lot1_01_activation_gate.py -q` → 9 passed +- `python -m scripts.verify_version_stamps --check` → ok + +## Acceptance criteria mapping + +- "close `not_authorized` in plan-only mode without production model code" — + satisfied exactly; no model/training file is touched. +- "no RSC/TwoTower path is reused as the treatment" — no model path is + touched at all. +- "no production default change" — satisfied. +- LOT1-02/LOT2/LOT3/LOT4 remain gated behind LOT1-01, unchanged. + +## Non-goals honored + +No K×c latent-workspace model code, no learned plan predictor, no causal +intervention campaign, no large training run or quality claim, no adaptive +depth/halting, no production default change. diff --git a/scripts/evaluate_lot1_01_activation_gate.py b/scripts/evaluate_lot1_01_activation_gate.py new file mode 100644 index 000000000..d7ea4c7a8 --- /dev/null +++ b/scripts/evaluate_lot1_01_activation_gate.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Evaluate SLM-250 (LOT1-01)'s hard activation gates in plan-only mode. + +No model, training, or K x c workspace code is loaded or executed. This +reads the two real committed upstream contract artifacts and emits the +required ``LotusOpenUIModelContractV1`` disposition. + +Example: + python -m scripts.evaluate_lot1_01_activation_gate \ + --fidelity-contract docs/design/lotus-openui-fidelity-contract-v1.json \ + --trace-gate-contract docs/design/compiler-reasoning-trace-v1.json \ + --out outputs/runs/slm250_activation_gate +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from slm_training.harnesses.experiments.lot1_01_activation_gate import ( + evaluate_activation_gates, + load_upstream_contract, + render_markdown, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="SLM-250 LOT1-01 hard-activation-gate evaluator (plan-only, no model code)" + ) + parser.add_argument( + "--fidelity-contract", + type=Path, + default=Path("docs/design/lotus-openui-fidelity-contract-v1.json"), + help="SLM-248 LotusOpenUIFidelityContractV1 JSON artifact", + ) + parser.add_argument( + "--trace-gate-contract", + type=Path, + default=Path("docs/design/compiler-reasoning-trace-v1.json"), + help="SLM-249 CompilerReasoningTraceGateV1 JSON artifact", + ) + parser.add_argument( + "--out", + type=Path, + default=Path("outputs/runs/slm250_activation_gate"), + ) + args = parser.parse_args(argv) + + fidelity_contract = load_upstream_contract(args.fidelity_contract) + trace_gate_contract = load_upstream_contract(args.trace_gate_contract) + + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + args.out.mkdir(parents=True, exist_ok=True) + contract.to_json(args.out / "lotus_openui_model_contract.json") + markdown = render_markdown(contract) + (args.out / "lotus_openui_model_contract.md").write_text(markdown, encoding="utf-8") + print(markdown) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/slm_training/harnesses/experiments/lot1_01_activation_gate.py b/src/slm_training/harnesses/experiments/lot1_01_activation_gate.py new file mode 100644 index 000000000..5599c26d1 --- /dev/null +++ b/src/slm_training/harnesses/experiments/lot1_01_activation_gate.py @@ -0,0 +1,241 @@ +"""SLM-250 LOT1-01 hard-activation-gate evaluator. + +LOT1-01 ("Implement the faithful causal K x c looped-latent model path") +declares two hard activation gates against its own upstream contracts: + +1. SLM-248 (LOT0-01) ``LotusOpenUIFidelityContractV1.authorization.verdict`` + must be ``authorize_bounded_implementation``. +2. SLM-249 (LOT0-02) ``CompilerReasoningTraceGateV1.gate.verdict`` must be + ``oracle_ceiling_positive`` (or another explicit authorization that + supplies K/c/stage targets). + +"Otherwise close ``not_authorized`` in plan-only mode without production +model code." This module evaluates those two gates against the real, +committed upstream contract artifacts (``docs/design/lotus-openui-fidelity-contract-v1.json`` +and ``docs/design/compiler-reasoning-trace-v1.json``) and emits the required +``LotusOpenUIModelContractV1`` disposition. It contains no model, training, +or K x c workspace code -- evaluating this gate honestly *is* the LOT1-01 +deliverable when the gate is unmet. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from slm_training.harness_core.versioning import build_version_stamp + +__all__ = [ + "AUTHORIZED_WIRING_ONLY", + "CONTRACT_ID", + "MODEL_CONTRACT_SCHEMA_VERSION", + "NOT_AUTHORIZED", + "REQUIRED_FIDELITY_VERDICT", + "REQUIRED_TRACE_VERDICT", + "GateEvaluation", + "LotusOpenUIModelContractV1", + "evaluate_activation_gates", + "load_upstream_contract", + "render_markdown", +] + +MODEL_CONTRACT_SCHEMA_VERSION = "lotus_openui_model_contract/v1" +CONTRACT_ID = "lotus-openui-model-contract-v1" + +# The exact verdict strings LOT1-01's own hard activation gates require. +REQUIRED_FIDELITY_VERDICT = "authorize_bounded_implementation" +REQUIRED_TRACE_VERDICT = "oracle_ceiling_positive" + +NOT_AUTHORIZED = "not_authorized" +AUTHORIZED_WIRING_ONLY = "authorized_wiring_only" + + +@dataclass(frozen=True) +class GateEvaluation: + """One hard-activation-gate check against a single upstream contract.""" + + gate_name: str + source_contract_id: str + source_contract_hash: str + source_linear_issue: str + required_verdict: str + actual_verdict: str + met: bool + rationale: str + + def to_dict(self) -> dict[str, Any]: + return dict(asdict(self)) + + +@dataclass(frozen=True) +class LotusOpenUIModelContractV1: + schema_version: str = MODEL_CONTRACT_SCHEMA_VERSION + contract_id: str = CONTRACT_ID + linear_issue: str = "SLM-250" + gate_1_fidelity_authorization: GateEvaluation | None = None + gate_2_trace_oracle_ceiling: GateEvaluation | None = None + verdict: str = NOT_AUTHORIZED + verdict_rationale: str = "" + version_stamp: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + data = dict(asdict(self)) + data["gate_1_fidelity_authorization"] = ( + self.gate_1_fidelity_authorization.to_dict() + if self.gate_1_fidelity_authorization is not None + else None + ) + data["gate_2_trace_oracle_ceiling"] = ( + self.gate_2_trace_oracle_ceiling.to_dict() + if self.gate_2_trace_oracle_ceiling is not None + else None + ) + data["contract_hash"] = self.contract_hash() + return data + + def to_json(self, path: Path) -> None: + path.write_text( + json.dumps(self.to_dict(), indent=2, sort_keys=True, default=str) + "\n", + encoding="utf-8", + ) + + def contract_hash(self) -> str: + payload = { + "schema_version": self.schema_version, + "contract_id": self.contract_id, + "linear_issue": self.linear_issue, + "gate_1_fidelity_authorization": ( + self.gate_1_fidelity_authorization.to_dict() + if self.gate_1_fidelity_authorization is not None + else None + ), + "gate_2_trace_oracle_ceiling": ( + self.gate_2_trace_oracle_ceiling.to_dict() + if self.gate_2_trace_oracle_ceiling is not None + else None + ), + "verdict": self.verdict, + } + return hashlib.sha256( + json.dumps(payload, sort_keys=True, default=str).encode("utf-8") + ).hexdigest() + + +def load_upstream_contract(path: Path) -> dict[str, Any]: + """Load an upstream contract JSON artifact from disk.""" + return json.loads(path.read_text(encoding="utf-8")) + + +def _evaluate_fidelity_gate(fidelity_contract: dict[str, Any]) -> GateEvaluation: + authorization = fidelity_contract.get("authorization") or {} + actual = authorization.get("verdict", "") + met = actual == REQUIRED_FIDELITY_VERDICT + return GateEvaluation( + gate_name="fidelity_contract_authorization", + source_contract_id=fidelity_contract.get("contract_id", ""), + source_contract_hash=fidelity_contract.get("contract_hash", ""), + source_linear_issue=authorization.get("linear_issue", "SLM-248"), + required_verdict=REQUIRED_FIDELITY_VERDICT, + actual_verdict=actual, + met=met, + rationale=authorization.get("verdict_rationale", ""), + ) + + +def _evaluate_trace_gate(trace_gate_contract: dict[str, Any]) -> GateEvaluation: + gate = trace_gate_contract.get("gate") or {} + actual = gate.get("verdict", "") + met = actual == REQUIRED_TRACE_VERDICT + return GateEvaluation( + gate_name="trace_oracle_ceiling", + source_contract_id=trace_gate_contract.get("contract_id", ""), + source_contract_hash=trace_gate_contract.get("contract_hash", ""), + source_linear_issue=gate.get("linear_issue", "SLM-249"), + required_verdict=REQUIRED_TRACE_VERDICT, + actual_verdict=actual, + met=met, + rationale=gate.get("verdict_rationale", ""), + ) + + +def evaluate_activation_gates( + fidelity_contract: dict[str, Any], + trace_gate_contract: dict[str, Any], +) -> LotusOpenUIModelContractV1: + """Evaluate LOT1-01's two hard activation gates against real upstream contracts. + + Returns a :class:`LotusOpenUIModelContractV1` whose ``verdict`` is + ``not_authorized`` unless both upstream gates report their exact required + verdict strings. No K x c model/training code path is implied or run by + a positive result; per the issue text, even an authorized gate delivers + "wiring/contract evidence only" (``authorized_wiring_only``), never a + semantic-quality claim. + """ + gate_1 = _evaluate_fidelity_gate(fidelity_contract) + gate_2 = _evaluate_trace_gate(trace_gate_contract) + + if gate_1.met and gate_2.met: + verdict = AUTHORIZED_WIRING_ONLY + rationale = ( + "Both hard activation gates report their required verdicts. " + "LOT1-01 may proceed in wiring/contract-evidence-only mode; " + "this contract authorizes no training or semantic-quality claim." + ) + else: + verdict = NOT_AUTHORIZED + unmet = [g.gate_name for g in (gate_1, gate_2) if not g.met] + rationale = ( + "Hard activation gate(s) unmet: " + + ", ".join(unmet) + + f". Gate 1 (SLM-248 fidelity authorization) reports verdict " + f"'{gate_1.actual_verdict}' (requires '{REQUIRED_FIDELITY_VERDICT}'). " + f"Gate 2 (SLM-249 trace oracle ceiling) reports verdict " + f"'{gate_2.actual_verdict}' (requires '{REQUIRED_TRACE_VERDICT}'). " + "Closing not_authorized in plan-only mode; no K x c model or " + "training code is added by this disposition." + ) + + return LotusOpenUIModelContractV1( + gate_1_fidelity_authorization=gate_1, + gate_2_trace_oracle_ceiling=gate_2, + verdict=verdict, + verdict_rationale=rationale, + version_stamp=build_version_stamp("harness.experiments"), + ) + + +def render_markdown(contract: LotusOpenUIModelContractV1) -> str: + g1 = contract.gate_1_fidelity_authorization + g2 = contract.gate_2_trace_oracle_ceiling + lines = [ + f"# SLM-250 LOT1-01 — Activation-gate disposition ({contract.contract_id})", + "", + f"Verdict: **{contract.verdict}**", + "", + contract.verdict_rationale, + "", + "## Gate 1 — fidelity contract authorization (SLM-248)", + "", + f"- Source contract: `{g1.source_contract_id}` (`{g1.source_contract_hash}`)", + f"- Required verdict: `{g1.required_verdict}`", + f"- Actual verdict: `{g1.actual_verdict}`", + f"- Met: **{g1.met}**", + "", + "## Gate 2 — trace oracle-ceiling (SLM-249)", + "", + f"- Source contract: `{g2.source_contract_id}` (`{g2.source_contract_hash}`)", + f"- Required verdict: `{g2.required_verdict}`", + f"- Actual verdict: `{g2.actual_verdict}`", + f"- Met: **{g2.met}**", + "", + "## Non-goals honored", + "", + "No K x c latent-workspace model code, no learned plan predictor, no " + "GPU training, no production default change. This disposition is " + "itself the LOT1-01 deliverable when the activation gate is unmet.", + "", + ] + return "\n".join(lines) diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index 311ba38b6..e4af0307c 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -1912,18 +1912,35 @@ ] }, "harness.experiments": { - "version": "v115", + "version": "v116", "kind": "harness", "paths": [ - "src/slm_training/harnesses/experiments/", "scripts/run_slm298_capacity_context_curriculum.py", + "src/slm_training/harnesses/experiments/", "tests/test_harnesses/experiments/test_slm298_capacity_context_curriculum.py", "tests/test_scripts/test_run_slm298_capacity_context_curriculum.py" ], "history": [ - {"version": "v115", "date": "2026-07-26", "note": "no-bump: migrate the CAP2 local-action fixture to TernaryECOCHead.entry_for without changing its action-code lookup."}, - {"version": "v115", "date": "2026-07-25", "note": "SLM-332 AP-009 binding contrast geometry harness and matched reconstruction control"}, - {"version": "v114", "date": "2026-07-25", "note": "SLM-332 fail-closed program-latent contrast, noise, and interpolation traversal primitives"}, + { + "version": "v116", + "date": "2026-07-25", + "note": "add SLM-250 LOT1-01 hard-activation-gate evaluator (lot1_01_activation_gate.py): reads SLM-248/SLM-249 real upstream contracts and closes not_authorized in plan-only mode, no K x c model/training code" + }, + { + "version": "v115", + "date": "2026-07-26", + "note": "no-bump: migrate the CAP2 local-action fixture to TernaryECOCHead.entry_for without changing its action-code lookup." + }, + { + "version": "v115", + "date": "2026-07-25", + "note": "SLM-332 AP-009 binding contrast geometry harness and matched reconstruction control" + }, + { + "version": "v114", + "date": "2026-07-25", + "note": "SLM-332 fail-closed program-latent contrast, noise, and interpolation traversal primitives" + }, { "version": "v113", "date": "2026-07-25", @@ -1954,6 +1971,11 @@ "date": "2026-07-25", "note": "add SLM-300 AP-015 self-context exposure-bias curriculum manifest, policy-origin-mixture wiring/fixture harness, and mixture-zero legacy-equivalence invariant" }, + { + "version": "v109", + "date": "2026-07-25", + "note": "add SLM-250 LOT1-01 hard-activation-gate evaluator (lot1_01_activation_gate.py): reads SLM-248/SLM-249 real upstream contracts and closes not_authorized in plan-only mode, no K x c model/training code" + }, { "version": "v108", "date": "2026-07-25", @@ -9520,11 +9542,31 @@ "tests/test_harnesses/test_slm332_latent_geometry.py" ], "history": [ - {"version": "v5", "date": "2026-07-25", "note": "normalize AgentV code-grader paths to portable repo URIs in committed evidence"}, - {"version": "v4", "date": "2026-07-25", "note": "retain declared opaque binding-target ordinals so AP-009 binding swaps survive factorization"}, - {"version": "v3", "date": "2026-07-25", "note": "add bounded local control-versus-combined runner with AgentEvals non-promotion assertion"}, - {"version": "v2", "date": "2026-07-25", "note": "wire admitted AP-009 binding pairs through the codec-only geometry harness"}, - {"version": "v1", "date": "2026-07-25", "note": "initial SLM-332 fail-closed latent geometry primitive"} + { + "version": "v5", + "date": "2026-07-25", + "note": "normalize AgentV code-grader paths to portable repo URIs in committed evidence" + }, + { + "version": "v4", + "date": "2026-07-25", + "note": "retain declared opaque binding-target ordinals so AP-009 binding swaps survive factorization" + }, + { + "version": "v3", + "date": "2026-07-25", + "note": "add bounded local control-versus-combined runner with AgentEvals non-promotion assertion" + }, + { + "version": "v2", + "date": "2026-07-25", + "note": "wire admitted AP-009 binding pairs through the codec-only geometry harness" + }, + { + "version": "v1", + "date": "2026-07-25", + "note": "initial SLM-332 fail-closed latent geometry primitive" + } ] }, "matrix.self_context_curriculum": { @@ -9694,6 +9736,25 @@ "note": "SLM-307: default-off packed block-structured bottleneck mask and masked SFT loss, with no-plan leakage and direct target-plan bypass canaries" } ] + }, + "harness.experiments.lot1_01_activation_gate": { + "version": "v1", + "kind": "gate", + "paths": [ + "src/slm_training/harnesses/experiments/lot1_01_activation_gate.py", + "scripts/evaluate_lot1_01_activation_gate.py", + "tests/test_harnesses/experiments/test_lot1_01_activation_gate.py", + "tests/test_scripts/test_evaluate_lot1_01_activation_gate.py", + "docs/design/iter-slm250-lot1-01-not-authorized-20260725.json", + "docs/design/iter-slm250-lot1-01-not-authorized-20260725.md" + ], + "history": [ + { + "version": "v1", + "date": "2026-07-25", + "note": "initial registration; SLM-250 LOT1-01 hard-activation-gate evaluator closes not_authorized against the real SLM-248/SLM-249 upstream contracts" + } + ] } } } diff --git a/tests/test_harnesses/experiments/test_lot1_01_activation_gate.py b/tests/test_harnesses/experiments/test_lot1_01_activation_gate.py new file mode 100644 index 000000000..085be80c7 --- /dev/null +++ b/tests/test_harnesses/experiments/test_lot1_01_activation_gate.py @@ -0,0 +1,138 @@ +"""Tests for slm_training.harnesses.experiments.lot1_01_activation_gate (SLM-250).""" + +from __future__ import annotations + +from pathlib import Path + +from slm_training.harnesses.experiments.lot1_01_activation_gate import ( + AUTHORIZED_WIRING_ONLY, + NOT_AUTHORIZED, + REQUIRED_FIDELITY_VERDICT, + REQUIRED_TRACE_VERDICT, + evaluate_activation_gates, + load_upstream_contract, + render_markdown, +) + +REPO_ROOT = Path(__file__).resolve().parents[3] +FIDELITY_CONTRACT_PATH = REPO_ROOT / "docs/design/lotus-openui-fidelity-contract-v1.json" +TRACE_GATE_CONTRACT_PATH = REPO_ROOT / "docs/design/compiler-reasoning-trace-v1.json" + + +def test_real_upstream_contracts_are_not_authorized() -> None: + """The actual, currently-committed SLM-248/SLM-249 verdicts must yield + not_authorized: SLM-248 reports needs_target_trace_contract (not + authorize_bounded_implementation) and SLM-249 reports inconclusive (not + oracle_ceiling_positive). This is the load-bearing proof that LOT1-01 + correctly closes without any K x c model/training code. + """ + fidelity_contract = load_upstream_contract(FIDELITY_CONTRACT_PATH) + trace_gate_contract = load_upstream_contract(TRACE_GATE_CONTRACT_PATH) + + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + assert contract.verdict == NOT_AUTHORIZED + assert contract.gate_1_fidelity_authorization.met is False + assert contract.gate_1_fidelity_authorization.actual_verdict == "needs_target_trace_contract" + assert contract.gate_2_trace_oracle_ceiling.met is False + assert contract.gate_2_trace_oracle_ceiling.actual_verdict == "inconclusive" + + +def test_both_gates_met_authorizes_wiring_only() -> None: + """The evaluator is not hardcoded to always fail: synthetic contracts + + reporting both required verdicts must flip the disposition, but only to + a wiring-only authorization, never a semantic-quality claim. + """ + fidelity_contract = { + "contract_id": "synthetic-fidelity", + "contract_hash": "deadbeef", + "authorization": { + "linear_issue": "SLM-248", + "verdict": REQUIRED_FIDELITY_VERDICT, + "verdict_rationale": "synthetic positive", + }, + } + trace_gate_contract = { + "contract_id": "synthetic-trace", + "contract_hash": "cafef00d", + "gate": { + "linear_issue": "SLM-249", + "verdict": REQUIRED_TRACE_VERDICT, + "verdict_rationale": "synthetic positive", + }, + } + + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + assert contract.verdict == AUTHORIZED_WIRING_ONLY + assert contract.gate_1_fidelity_authorization.met is True + assert contract.gate_2_trace_oracle_ceiling.met is True + + +def test_only_one_gate_met_stays_not_authorized() -> None: + fidelity_contract = { + "contract_id": "synthetic-fidelity", + "contract_hash": "deadbeef", + "authorization": {"linear_issue": "SLM-248", "verdict": REQUIRED_FIDELITY_VERDICT}, + } + trace_gate_contract = { + "contract_id": "synthetic-trace", + "contract_hash": "cafef00d", + "gate": {"linear_issue": "SLM-249", "verdict": "inconclusive"}, + } + + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + assert contract.verdict == NOT_AUTHORIZED + assert contract.gate_1_fidelity_authorization.met is True + assert contract.gate_2_trace_oracle_ceiling.met is False + + +def test_missing_fields_fail_closed_not_authorized() -> None: + contract = evaluate_activation_gates({}, {}) + assert contract.verdict == NOT_AUTHORIZED + assert contract.gate_1_fidelity_authorization.actual_verdict == "" + assert contract.gate_2_trace_oracle_ceiling.actual_verdict == "" + + +def test_contract_hash_is_stable_and_changes_with_verdict() -> None: + fidelity_contract = load_upstream_contract(FIDELITY_CONTRACT_PATH) + trace_gate_contract = load_upstream_contract(TRACE_GATE_CONTRACT_PATH) + + a = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + b = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + assert a.contract_hash() == b.contract_hash() + + synthetic_fidelity = dict(fidelity_contract) + synthetic_fidelity["authorization"] = dict(fidelity_contract["authorization"]) + synthetic_fidelity["authorization"]["verdict"] = REQUIRED_FIDELITY_VERDICT + c = evaluate_activation_gates(synthetic_fidelity, trace_gate_contract) + assert c.contract_hash() != a.contract_hash() + + +def test_to_dict_and_to_json_round_trip(tmp_path: Path) -> None: + fidelity_contract = load_upstream_contract(FIDELITY_CONTRACT_PATH) + trace_gate_contract = load_upstream_contract(TRACE_GATE_CONTRACT_PATH) + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + out_path = tmp_path / "contract.json" + contract.to_json(out_path) + assert out_path.exists() + + data = contract.to_dict() + assert data["verdict"] == NOT_AUTHORIZED + assert data["gate_1_fidelity_authorization"]["source_contract_id"] == "lotus-openui-fidelity-contract-v1" + assert data["gate_2_trace_oracle_ceiling"]["source_contract_id"] == "compiler-reasoning-trace-v1" + + +def test_render_markdown_includes_verdict_and_gates() -> None: + fidelity_contract = load_upstream_contract(FIDELITY_CONTRACT_PATH) + trace_gate_contract = load_upstream_contract(TRACE_GATE_CONTRACT_PATH) + contract = evaluate_activation_gates(fidelity_contract, trace_gate_contract) + + md = render_markdown(contract) + assert "not_authorized" in md + assert "needs_target_trace_contract" in md + assert "inconclusive" in md + assert "No K x c latent-workspace model code" in md diff --git a/tests/test_scripts/test_evaluate_lot1_01_activation_gate.py b/tests/test_scripts/test_evaluate_lot1_01_activation_gate.py new file mode 100644 index 000000000..b64faa841 --- /dev/null +++ b/tests/test_scripts/test_evaluate_lot1_01_activation_gate.py @@ -0,0 +1,36 @@ +"""Tests for scripts/evaluate_lot1_01_activation_gate.py (SLM-250).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from scripts import evaluate_lot1_01_activation_gate + + +def test_default_run_reports_not_authorized(tmp_path: Path) -> None: + out = tmp_path / "gate" + rc = evaluate_lot1_01_activation_gate.main(["--out", str(out)]) + assert rc == 0 + + data = json.loads((out / "lotus_openui_model_contract.json").read_text()) + assert data["verdict"] == "not_authorized" + assert (out / "lotus_openui_model_contract.md").exists() + + +def test_run_against_explicit_contract_paths(tmp_path: Path) -> None: + out = tmp_path / "gate_explicit" + rc = evaluate_lot1_01_activation_gate.main( + [ + "--fidelity-contract", + "docs/design/lotus-openui-fidelity-contract-v1.json", + "--trace-gate-contract", + "docs/design/compiler-reasoning-trace-v1.json", + "--out", + str(out), + ] + ) + assert rc == 0 + markdown = (out / "lotus_openui_model_contract.md").read_text() + assert "SLM-250" in markdown + assert "not_authorized" in markdown