From 40465ded915078cdafa01280bf5e6fae81c37490 Mon Sep 17 00:00:00 2001 From: Amber Li Date: Thu, 20 Aug 2026 15:04:52 -0400 Subject: [PATCH 1/2] sysID: explainability is a question about structure, not about magnitude The trimmer drops a recording no parameter value can explain, so a chaotic replay cannot drag the fit to a compromise nobody supports. It decides with an RMS bar: trim_rms_factor * noise_sigma = 0.1. Under interval scoring that bar is applied to two residuals that are nothing alike. A MISSING-CASCADE PENALTY says the twin failed to reproduce WHICH dominoes fall. A timing term says both sides saw a domino fall, at different moments. On run_20260820_141450 one penalty was 2884 in SSE while the entire timing disagreement was 5.3 -- so the bar is really a penalty detector with a wildly miscalibrated tail, and it rejects structurally perfect data: per-trajectory best RMS ['inf', 'inf', '1.21'] vs threshold 0.1000 NO trajectory is explainable at any candidate params That 1.21 is a theta at which the twin reproduces ALL FOUR falls with zero penalties -- the same theta the sweep reports as explaining the data 1084x better than baseline (0.2402 -> 5.321 against 5769 at the agent's belief). The fit refused to run on the best explanation it had, because the residual timing error the fit exists to REDUCE was used as grounds for refusing to fit. That is circular. So ask the question that was meant: did any candidate reproduce the cascade? Three outcomes rather than one number, and the middle one is the whole point: nothing measured at all -> not explainable (nothing to explain) any missing-cascade penalty -> not explainable (wrong dominoes fall) penalties absent, timing only -> EXPLAINABLE, and the fit does its job The first row is not padding. A cascade-free segment emits no penalties because it emits nothing, and reading "no penalties" as success there would reinstate exactly the inversion that made an empty residual vector an RMS of 0.0 -- vacuous segments outranking real ones. Counted from the id sets (a penalty IS a domino one side has and the other does not, which is the symmetric difference), never by recognising penalty-sized numbers in the output: magnitude-sniffing works until a real disagreement lands on the penalty value. THE PER-STEP OBJECTIVE IS UNTOUCHED. There the residuals really are a dimensionless fraction of typical motion, compute_residual_scaling put them there, and 0.1 means what it says -- so the RMS bar still decides, gated on score_observed_only. Scope: reproduced_cascade asks whether ANY candidate on the grid reproduced the cascade, not whether the RMS-best one did. Those differ -- a theta that completes the chain but mistimes it can score worse than one that stalls it early with fewer terms -- and "could this model ever produce this outcome" is a question about the whole grid. This IS more permissive than before: a recording where the twin gets the right dominoes falling but badly wrong timing is now kept and drives the fit. That is the data the fit should use, but it is a real behaviour change and not a refactor. min_explainable_fits gains a third return and the explainability cache a third element; the stub in test_trimming_uses_rms_cache moves with them. The new test fails under a revert of the structural switch, and exactly that one test fails -- it passes its own SysIdConfig rather than touching CFG, so interval scoring does not leak into the rest of the module. --- predicators/code_sim_learning/grid_seed.py | 47 +++--- predicators/code_sim_learning/orchestrator.py | 8 +- .../code_sim_learning/physical_sysid.py | 61 ++++++-- .../code_sim_learning/rollout_objective.py | 139 ++++++++++++++++-- .../code_sim_learning/test_physical_sysid.py | 91 +++++++++++- 5 files changed, 291 insertions(+), 55 deletions(-) diff --git a/predicators/code_sim_learning/grid_seed.py b/predicators/code_sim_learning/grid_seed.py index f4b751227..ff1d8c46a 100644 --- a/predicators/code_sim_learning/grid_seed.py +++ b/predicators/code_sim_learning/grid_seed.py @@ -20,7 +20,7 @@ scalar_from_fit_space, scalar_to_fit_space from predicators.code_sim_learning.rollout_env import RolloutTrajectory from predicators.code_sim_learning.rollout_objective import \ - compute_rollout_sse, per_trajectory_rms + compute_rollout_sse, per_trajectory_scores from predicators.code_sim_learning.trajectory_prep import ResidualScaling logger = logging.getLogger(__name__) @@ -351,17 +351,17 @@ def min_explainable_rms( and unexplainable as the agent re-declared inits across calls (observed on run_20260711_141026, cycle 2). """ - best, _ = min_explainable_fits(base_env, - trajectories, - physical_specs, - residual_features, - rules=rules, - rule_specs=rule_specs, - latent_init=latent_init, - extra_candidates=extra_candidates, - scaling=scaling, - anchors=anchors, - config=config) + best, _, _ = min_explainable_fits(base_env, + trajectories, + physical_specs, + residual_features, + rules=rules, + rule_specs=rule_specs, + latent_init=latent_init, + extra_candidates=extra_candidates, + scaling=scaling, + anchors=anchors, + config=config) return best @@ -377,7 +377,7 @@ def min_explainable_fits( scaling: Optional[ResidualScaling] = None, anchors: Optional[Dict[str, float]] = None, config: Optional[SysIdConfig] = None, -) -> Tuple[List[float], List[Dict[str, float]]]: +) -> Tuple[List[float], List[Dict[str, float]], List[bool]]: """:func:`min_explainable_rms` plus each trajectory's argmin params. The second return is, per trajectory, the PHYSICAL portion of the @@ -412,12 +412,19 @@ def min_explainable_fits( n: float(base[n]) for n in physical_names } for _ in trajectories] + # Whether ANY candidate reproduced the cascade -- not whether the + # RMS-best one did. The two differ: under interval scoring a theta that + # completes the chain but mistimes it can score worse than one that + # stalls it early with few terms, and "could this model ever produce + # this outcome" is a question about the whole grid. + reproduced = [False] * len(trajectories) for params in candidates: - rms = per_trajectory_rms(base_env, trajectories, params, - residual_features, physical_names, rules, - latent_init, scaling) - for i, r in enumerate(rms): - if r < best[i]: - best[i] = r + scores = per_trajectory_scores(base_env, trajectories, params, + residual_features, physical_names, + rules, latent_init, scaling) + for i, score in enumerate(scores): + reproduced[i] = reproduced[i] or score.stats.reproduced_cascade + if score.rms < best[i]: + best[i] = score.rms best_params[i] = {n: float(params[n]) for n in physical_names} - return best, best_params + return best, best_params, reproduced diff --git a/predicators/code_sim_learning/orchestrator.py b/predicators/code_sim_learning/orchestrator.py index 80d184e1d..4642e9529 100644 --- a/predicators/code_sim_learning/orchestrator.py +++ b/predicators/code_sim_learning/orchestrator.py @@ -104,8 +104,8 @@ def run_rollout_sysid( rule_specs: Sequence[ParamSpec] = (), latent_init: Any = None, anchors: Optional[Dict[str, float]] = None, - rms_cache: Optional[Dict[Tuple, Tuple[List[float], - List[Dict[str, float]]]]] = None, + rms_cache: Optional[Dict[Tuple, Tuple[List[float], List[Dict[str, float]], + List[bool]]]] = None, fit_cache: Optional[Dict[Tuple, _FitComputation]] = None, fit_cache_key: Optional[Any] = None, report_adjuster: Optional[ReportAdjuster] = None, @@ -239,8 +239,8 @@ def _compute_fit( rule_specs: Sequence[ParamSpec], latent_init: Any, anchors: Dict[str, float], - rms_cache: Optional[Dict[Tuple, Tuple[List[float], List[Dict[str, - float]]]]], + rms_cache: Optional[Dict[Tuple, Tuple[List[float], List[Dict[str, float]], + List[bool]]]], config: SysIdConfig, ) -> _FitComputation: """The cacheable fit core: trim + fit + report on the survivors.""" diff --git a/predicators/code_sim_learning/physical_sysid.py b/predicators/code_sim_learning/physical_sysid.py index b7422ef39..0a493e9c6 100644 --- a/predicators/code_sim_learning/physical_sysid.py +++ b/predicators/code_sim_learning/physical_sysid.py @@ -623,8 +623,8 @@ def fit_params_rollout_trimmed( noise_sigma: float = 0.05, scaling: Optional[ResidualScaling] = None, anchors: Optional[Dict[str, float]] = None, - rms_cache: Optional[Dict[Tuple, Tuple[List[float], - List[Dict[str, float]]]]] = None, + rms_cache: Optional[Dict[Tuple, Tuple[List[float], List[Dict[str, float]], + List[bool]]]] = None, config: Optional[SysIdConfig] = None, ) -> Tuple[FitResult, List[RolloutTrajectory], List[float], List[Dict[str, float]]]: @@ -695,7 +695,8 @@ def fit_params_rollout_trimmed( config=config) return result, list(trajectories), [], [] cache_key: Optional[Tuple] = None - cached: Optional[Tuple[List[float], List[Dict[str, float]]]] = None + cached: Optional[Tuple[List[float], List[Dict[str, float]], + List[bool]]] = None if rms_cache is not None: cache_key = _explainability_cache_key(physical_specs, rule_specs, trajectories, anchors, scaling, @@ -725,17 +726,51 @@ def fit_params_rollout_trimmed( "%.1fs.", num_rollouts_run() - sweep_n0, time.monotonic() - sweep_t0) - rms, argmins = cached + rms, argmins, reproduced = cached threshold = factor * noise_sigma - survivors = [t for t, r in zip(trajectories, rms) if r <= threshold] - surv_argmins = [a for a, r in zip(argmins, rms) if r <= threshold] + # UNDER INTERVAL SCORING THE QUESTION IS STRUCTURAL, NOT NUMERIC. An RMS + # bar asks "how closely does the twin match", but the residuals it is + # applied to are a missing-cascade penalty (the twin did not reproduce + # WHICH dominoes fall) and a timing difference (both fell, at different + # moments) added together -- and those differ by three orders of + # magnitude, so the bar is really a penalty detector with a + # wildly-miscalibrated tail. On run_20260820_141450 a theta reproducing + # all four falls with ZERO penalties scored 1.21 against the 0.1 bar and + # was dropped as "unexplainable at any candidate params", while the same + # theta was reported by the sweep as explaining the data 1084x better + # than baseline. + # + # So ask the question that was meant: did any candidate reproduce the + # cascade? The leftover timing error is precisely what the fit exists to + # reduce, and using it to refuse to fit is circular. The per-step + # objective keeps the RMS bar, where residuals really are a + # dimensionless fraction of typical motion and the bar means what it says. + structural = config.score_observed_only + + def _explainable(index: int) -> bool: + """Whether trajectory ``index`` is worth fitting on.""" + if structural: + return reproduced[index] + return rms[index] <= threshold + + keep = [i for i in range(len(trajectories)) if _explainable(i)] + survivors = [trajectories[i] for i in keep] + surv_argmins = [argmins[i] for i in keep] if len(survivors) < len(trajectories): - logger.info( - "Rollout sysID trimming: per-trajectory best RMS %s vs " - "threshold %.4f (%g x noise %.3f) — dropping %d of %d " - "unexplainable trajectories.", [f"{r:.4g}" for r in rms], - threshold, factor, noise_sigma, - len(trajectories) - len(survivors), len(trajectories)) + if structural: + logger.info( + "Rollout sysID trimming: no candidate reproduced the cascade " + "for %d of %d trajectories (best RMS %s, kept for the fit " + "rather than compared against a bar) — dropping them.", + len(trajectories) - len(survivors), len(trajectories), + [f"{r:.4g}" for r in rms]) + else: + logger.info( + "Rollout sysID trimming: per-trajectory best RMS %s vs " + "threshold %.4f (%g x noise %.3f) — dropping %d of %d " + "unexplainable trajectories.", [f"{r:.4g}" for r in rms], + threshold, factor, noise_sigma, + len(trajectories) - len(survivors), len(trajectories)) if not survivors: logger.warning( "Rollout sysID trimming: NO trajectory is explainable at any " @@ -755,7 +790,7 @@ def fit_params_rollout_trimmed( # cleanest data rather than the loudest. consistency = config.consistency_factor physical_names = [s.name for s in physical_specs] - best = [r for r in rms if r <= threshold] + best = [rms[i] for i in keep] hull_candidates: List[Dict[str, float]] = [] while True: result = fit_params_rollout(base_env, diff --git a/predicators/code_sim_learning/rollout_objective.py b/predicators/code_sim_learning/rollout_objective.py index 1ccca5c14..0d35ea8fd 100644 --- a/predicators/code_sim_learning/rollout_objective.py +++ b/predicators/code_sim_learning/rollout_objective.py @@ -65,6 +65,44 @@ def reset_track_cache() -> None: _TRACK_CACHE.clear() +@dataclasses.dataclass +class IntervalStats: + """What an interval-scored evaluation was made of, not just how big. + + An RMS collapses two failures that are nothing alike. A + MISSING-CASCADE PENALTY says the twin did not reproduce which + dominoes fall; a timing term says both fell, at different moments. + On run_20260820_123606 one penalty was 2884 in SSE while the entire + timing disagreement was 5.3, so any single penalty buries the + timing -- and a structurally perfect reproduction with imperfect + timing still scored 1.3, well above the 0.1 trim bar. + + Deciding whether a recording is explainable AT ALL is a question + about structure, so the counts are reported separately and the + caller asks the question it actually means. Filled in only by the + interval paths; the per-step objective leaves it at zero. + """ + + terms: int = 0 + penalties: int = 0 + + @property + def measured(self) -> bool: + """Whether anything was scored here at all.""" + return self.terms > 0 + + @property + def reproduced_cascade(self) -> bool: + """Whether every domino one side saw fall, the other saw too. + + False when nothing was measured: there is no cascade to have + reproduced, and reading "no penalties" as success would let a + segment that scored nothing pass as ideal evidence -- the same + inversion that made an empty residual vector an RMS of 0.0. + """ + return self.measured and self.penalties == 0 + + def compute_rollout_sse( base_env: Any, trajectories: List[RolloutTrajectory], @@ -114,6 +152,7 @@ def compute_rollout_residuals( scaling: Optional[ResidualScaling] = None, config: Optional[SysIdConfig] = None, episode_count: Optional[int] = None, + stats: Optional[IntervalStats] = None, ) -> np.ndarray: """Rollout residuals (predicted - observed, scaled) as a flat vector. @@ -125,12 +164,18 @@ def compute_rollout_residuals( ``episode_count`` is how many trajectories the CALLER holds, when that differs from how many were passed here: see :func:`_iter_rollout_residual_terms`. + + ``stats``, when given, is FILLED IN with what the interval terms + were made of -- see :class:`IntervalStats`. The vector alone cannot + say whether a large residual is a cascade the twin failed to + reproduce or one it reproduced with the wrong timing, and those two + call for opposite decisions. """ return np.asarray(list( _iter_rollout_residual_terms(base_env, trajectories, params, residual_features, physical_names, rules, latent_init, scaling, config, - episode_count)), + episode_count, stats)), dtype=float) @@ -167,6 +212,7 @@ def _iter_rollout_residual_terms( scaling: Optional[ResidualScaling] = None, config: Optional[SysIdConfig] = None, episode_count: Optional[int] = None, + stats: Optional[IntervalStats] = None, ) -> Iterator[float]: """Yield per-feature residuals for the joint forward model. @@ -281,7 +327,7 @@ def _run_rules_post_step(env: Any, sim_state: State, i: int) -> None: yield from _interval_residual_terms(sim_states, states, tracks[traj_index], id_maps[traj_index], - config, summary_w) + config, summary_w, stats) else: # Segments of ONE episode. Held, not scored: see the episode # -level yield after this loop. @@ -325,7 +371,7 @@ def _run_rules_post_step(env: Any, sim_state: State, i: int) -> None: yield from _episode_interval_terms(episode_rollouts, [s for s, _ in trajectories], tracks[-1], id_maps[0], config, - summary_w) + summary_w, stats) def _load_scored_track(config: SysIdConfig) -> Optional[Any]: @@ -498,6 +544,24 @@ def _map_for(states: List[State], track: Any) -> Dict[str, int]: return [shared] * len(trajectories) +def _record_interval_stats(stats: Optional[IntervalStats], + sim_intervals: Dict[int, float], + obs_intervals: Dict[int, float]) -> None: + """Count what ``interval_residuals`` is about to emit, by kind. + + Counted from the id sets rather than by recognising penalty-sized + numbers in the output: a penalty is exactly a domino one side has + and the other does not, which is what the symmetric difference is. + Sniffing magnitudes would work today and break the moment a real + disagreement happened to land on the penalty value. + """ + if stats is None: + return + ids = set(sim_intervals) | set(obs_intervals) + stats.terms += len(ids) + stats.penalties += len(set(sim_intervals) ^ set(obs_intervals)) + + def _interval_scale(obs_intervals: Dict[int, float], penalty: float) -> float: """Divisor putting interval residuals in the units everything expects. @@ -534,10 +598,14 @@ def _interval_scale(obs_intervals: Dict[int, float], penalty: float) -> float: return penalty if penalty > 0.0 else 1.0 -def _episode_interval_terms(rollouts: List[List[State]], - recorded: List[List[State]], track: Any, - name_to_id: Dict[str, int], config: SysIdConfig, - summary_w: float) -> Iterator[float]: +def _episode_interval_terms( + rollouts: List[List[State]], + recorded: List[List[State]], + track: Any, + name_to_id: Dict[str, int], + config: SysIdConfig, + summary_w: float, + stats: Optional[IntervalStats] = None) -> Iterator[float]: """Yield ONE set of propagation-interval residuals for a whole episode. Rest-point segmentation is a ROLLOUT device -- multiple shooting, to @@ -615,16 +683,21 @@ def _onsets(series: Any) -> Dict[int, float]: total_steps = sum(len(s) for s in rollouts) penalty = max(track.duration_s, step_s * total_steps) del recorded # kept in the signature for symmetry with the per-segment path + _record_interval_stats(stats, sim_intervals, obs_intervals) scale = _interval_scale(obs_intervals, penalty) for res in interval_residuals(sim_intervals, obs_intervals, penalty, scale): yield summary_w * res -def _interval_residual_terms(sim_states: List[State], recorded: List[State], - track: Any, name_to_id: Dict[str, int], - config: SysIdConfig, - summary_w: float) -> Iterator[float]: +def _interval_residual_terms( + sim_states: List[State], + recorded: List[State], + track: Any, + name_to_id: Dict[str, int], + config: SysIdConfig, + summary_w: float, + stats: Optional[IntervalStats] = None) -> Iterator[float]: """Yield (sim - observed) propagation intervals, in seconds. The residual set for a whole trajectory is one term per domino @@ -732,6 +805,7 @@ def _onsets(series: Any) -> Dict[int, float]: # evidence there is, so the stand-in is the track's own span rather than # a small number: it must cost more than any real disagreement. penalty = max(track.duration_s, step_s * len(sim_states)) + _record_interval_stats(stats, sim_intervals, obs_intervals) scale = _interval_scale(obs_intervals, penalty) for res in interval_residuals(sim_intervals, obs_intervals, penalty, scale): @@ -883,8 +957,42 @@ def per_trajectory_rms( of length 1; the full count goes down as ``episode_count`` so the objective can still tell an episode from a segment of one. """ - out: List[float] = [] + return [ + score.rms for score in per_trajectory_scores( + base_env, trajectories, params, residual_features, physical_names, + rules, latent_init, scaling, config) + ] + + +@dataclasses.dataclass +class TrajectoryScore: + """One trajectory's fit at one theta: how big, and made of what.""" + + rms: float + stats: IntervalStats + + +def per_trajectory_scores( + base_env: Any, + trajectories: List[RolloutTrajectory], + params: Dict[str, float], + residual_features: Dict[str, List[str]], + physical_names: Sequence[str], + rules: Sequence[Any] = (), + latent_init: Any = None, + scaling: Optional[ResidualScaling] = None, + config: Optional[SysIdConfig] = None, +) -> List[TrajectoryScore]: + """:func:`per_trajectory_rms`, keeping what the RMS was made of. + + The magnitude answers "how well does this theta fit"; the stats + answer "did this theta reproduce the cascade at all", and the + trimmer needs the second question rather than the first. See + :class:`IntervalStats`. + """ + out: List[TrajectoryScore] = [] for traj in trajectories: + stats = IntervalStats() res = compute_rollout_residuals(base_env, [traj], params, residual_features, @@ -893,7 +1001,8 @@ def per_trajectory_rms( latent_init, scaling, config, - episode_count=len(trajectories)) - out.append( - float(np.sqrt(np.mean(res**2))) if res.size else float("inf")) + episode_count=len(trajectories), + stats=stats) + rms = float(np.sqrt(np.mean(res**2))) if res.size else float("inf") + out.append(TrajectoryScore(rms=rms, stats=stats)) return out diff --git a/tests/code_sim_learning/test_physical_sysid.py b/tests/code_sim_learning/test_physical_sysid.py index 667cb4f12..b085a1bfc 100644 --- a/tests/code_sim_learning/test_physical_sysid.py +++ b/tests/code_sim_learning/test_physical_sysid.py @@ -28,6 +28,25 @@ _RESIDUAL_FEATURES = {"domino": ["x"]} +def _scores_from(rms_fn): + """Wrap an RMS-only stub as the per-trajectory scores the sweep wants. + + The explainability sweep asks for structure as well as magnitude + (see ``rollout_objective.IntervalStats``). These stubs cover the + PER-STEP objective, where the RMS bar still decides and the interval + stats are never consulted, so they come back empty. + """ + + def _scores(env, trajectories, *args, **kwargs): + return [ + rollout_objective.TrajectoryScore( + rms=r, stats=rollout_objective.IntervalStats()) + for r in rms_fn(env, trajectories, *args, **kwargs) + ] + + return _scores + + def _trajectory(domino_xs, robot_xs=None): """Build a (states, actions) trajectory from per-step feature values.""" domino = Object("d0", _DOMINO_TYPE) @@ -504,7 +523,8 @@ def fake_rms(_env, trajectories, *_args, **_kwargs): monkeypatch.setattr(physical_sysid, "per_trajectory_rms", fake_rms) # min_explainable_rms (called by the trimmed fit) evaluates its # candidate grid through grid_seed's namespace. - monkeypatch.setattr(grid_seed, "per_trajectory_rms", fake_rms) + monkeypatch.setattr(grid_seed, "per_trajectory_scores", + _scores_from(fake_rms)) # The stub trajectories are plain strings; skip the data-derived # residual scaling (exercised by its own tests). monkeypatch.setattr(physical_sysid, "compute_residual_scaling", @@ -530,6 +550,70 @@ def test_trimming_drops_unexplainable_and_refits(monkeypatch): assert result.point_estimate["friction"] == 0.1 +def test_a_reproduced_cascade_is_explainable_however_bad_its_timing( + monkeypatch): + """Under interval scoring, explainability is structural, not numeric. + + An RMS bar cannot tell a cascade the twin FAILED TO REPRODUCE from + one it reproduced with the wrong timing, and the two differ by three + orders of magnitude, so the bar is really a penalty detector with a + miscalibrated tail. On run_20260820_141450 a theta that reproduced + all four falls with zero penalties scored 1.21 against the 0.1 bar + and was dropped as "unexplainable at any candidate params" -- while + the sweep reported that same theta as explaining the data 1084x + better than baseline. The leftover timing error is what the fit + exists to reduce; refusing to fit because of it is circular. + """ + # pylint: disable-next=import-outside-toplevel + import dataclasses + + # Passed in rather than set on CFG: a reset_config here leaks + # interval scoring into every later test in this module. + config = dataclasses.replace(physical_sysid.SysIdConfig.from_cfg(), + score_observed_only=True) + specs = [ParamSpec("friction", 0.5, lo=0.01, hi=2)] + # empty: nothing measured. cascade: reproduced, badly timed. stalled: + # the chain never reached two dominoes at any candidate. + trajs = ["empty", "cascade", "stalled"] + stats_by_name = { + "empty": rollout_objective.IntervalStats(terms=0, penalties=0), + "cascade": rollout_objective.IntervalStats(terms=4, penalties=0), + "stalled": rollout_objective.IntervalStats(terms=4, penalties=2), + } + rms_by_name = {"empty": float("inf"), "cascade": 1.21, "stalled": 5000.0} + + def fake_scores(_env, trajectories, *_a, **_k): + return [ + rollout_objective.TrajectoryScore(rms=rms_by_name[t], + stats=stats_by_name[t]) + for t in trajectories + ] + + def fake_fit(_env, _trajectories, *_args, **_kwargs): + return FitResult(names=["friction"], + samples=np.array([[0.24]], dtype=float), + log_probs=np.zeros(1), + jacobian=None, + noise_sigma=0.05, + prior_sigma=np.array([0.375])) + + monkeypatch.setattr(grid_seed, "per_trajectory_scores", fake_scores) + monkeypatch.setattr(physical_sysid, "fit_params_rollout", fake_fit) + monkeypatch.setattr( + physical_sysid, "per_trajectory_rms", lambda _e, trajectories, *_a, ** + _k: [rms_by_name[t] for t in trajectories]) + monkeypatch.setattr(physical_sysid, "compute_residual_scaling", + lambda *_a, **_k: None) + + _result, survivors, _rms, _hull = fit_params_rollout_trimmed( + None, trajs, specs, {"domino": ["x"]}, config=config) + + assert survivors == ["cascade"], \ + "the segment that reproduced the cascade must be fitted on, at " \ + "1.21 against a 0.1 bar; the empty one measured nothing and the " \ + "stalled one never reproduced the chain" + + def test_trimming_keeps_all_when_explainable(monkeypatch): """No trajectory above threshold -> single fit, nothing dropped.""" specs = [ParamSpec("friction", 0.5, lo=0.01, hi=2)] @@ -580,7 +664,8 @@ def fake_rms(_env, trajectories, *_args, **_kwargs): monkeypatch.setattr(physical_sysid, "fit_params_rollout", fake_fit) monkeypatch.setattr(physical_sysid, "per_trajectory_rms", fake_rms) - monkeypatch.setattr(grid_seed, "per_trajectory_rms", fake_rms) + monkeypatch.setattr(grid_seed, "per_trajectory_scores", + _scores_from(fake_rms)) monkeypatch.setattr(physical_sysid, "compute_residual_scaling", lambda *_a, **_k: None) result, survivors, _rms, hull = fit_params_rollout_trimmed( @@ -1043,7 +1128,7 @@ def fake_min_fits(_env, trajectories, *_args, **_kwargs): calls["sweep"] += 1 return ([0.001] * len(trajectories), [{ "friction": 0.1 - } for _ in trajectories]) + } for _ in trajectories], [True] * len(trajectories)) def fake_fit(_env, _trajectories, *_args, **_kwargs): return FitResult(names=["friction"], From 37d67c6dcd50cb530a7f0f540164cdab1a0b560a Mon Sep 17 00:00:00 2001 From: Amber Li Date: Thu, 20 Aug 2026 16:50:37 -0400 Subject: [PATCH 2/2] sysID: carry the third explainability verdict through the typing contract The explainability cache gained a third element (whether any candidate reproduced the segment's cascade) but two DECLARATIONS of its type did not follow: the approach's own attribute, and SynthesisBackend, the structural protocol the synthesis tools are typed against. A protocol mismatch does not surface where the attribute is declared -- it surfaces as the whole approach failing to satisfy the protocol at the call sites that pass it, which is why the error named create_predicate_synthesis_tools and not the cache. Caught by CI, not locally: I ran mypy over predicators/code_sim_learning/ and the failure was three directories away. CI runs `mypy . --config-file mypy.ini` over 721 files, and so does this commit's verification. --- predicators/agent_sdk/synthesis_backend.py | 10 ++++++++-- predicators/approaches/agent_sim_learning_approach.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/predicators/agent_sdk/synthesis_backend.py b/predicators/agent_sdk/synthesis_backend.py index 794057fef..1d85e46f0 100644 --- a/predicators/agent_sdk/synthesis_backend.py +++ b/predicators/agent_sdk/synthesis_backend.py @@ -52,9 +52,15 @@ class SynthesisBackend(Protocol): _types: Set[Type] _initial_predicates: Set[Predicate] _initial_options: Set[ParameterizedOption] - # Per-fit cache of best-achievable per-segment RMS (system ID). + # Per-fit cache of the explainability sweep's verdicts (system ID): + # best-achievable per-segment RMS, each segment's argmin params, and + # whether ANY candidate reproduced its cascade -- the last being what + # decides explainability under interval scoring, where an RMS bar + # cannot separate "the twin did not reproduce which dominoes fall" + # from "it reproduced them with the wrong timing". _explainability_cache: Dict[Tuple, Tuple[List[float], List[Dict[str, - float]]]] + float]], + List[bool]]] # ── State written by the tools ─────────────────────────────── # Per-skill samplers keyed by option name. diff --git a/predicators/approaches/agent_sim_learning_approach.py b/predicators/approaches/agent_sim_learning_approach.py index d34592326..eee797348 100644 --- a/predicators/approaches/agent_sim_learning_approach.py +++ b/predicators/approaches/agent_sim_learning_approach.py @@ -804,8 +804,8 @@ def __init__(self, # reuse the sweep instead of re-rolling it, which both saves # rollouts and pins the verdict for identical inputs. self._explainability_cache: Dict[Tuple, Tuple[List[float], - List[Dict[str, - float]]]] = {} + List[Dict[str, float]], + List[bool]]] = {} # Whole-fit memoization for the orchestrator (same lifecycle as # the explainability cache): repeated canonical sim.fit calls on # an unchanged artifact version + data reuse the entire fit core