diff --git a/docs/design/var1-02-set-property-action-20260726.json b/docs/design/var1-02-set-property-action-20260726.json new file mode 100644 index 000000000..b35668ba4 --- /dev/null +++ b/docs/design/var1-02-set-property-action-20260726.json @@ -0,0 +1,841 @@ +{ + "schema": "var1_02_set_property_action/v1", + "issue": "SLM-425 (VAR1-02)", + "gated_on": "docs/design/var1-01-set-property-probe-20260725.md (SLM-424/VAR1-01)", + "hypothetical": false, + "production_action_space_changed": true, + "generated_at": "2026-07-26T19:54:14Z", + "mode": "extended", + "max_edits": 8, + "node_budget": 15, + "seed_source": "root = Stack([], \"column\")", + "action_added": "ACTION_SET_PROPERTY", + "n_actions": 12, + "checkpoint_format": 3, + "kernel_op": "openui.set_property", + "verdict_policy": "reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim.", + "production_report": { + "schema": "slm299_edit_reachability_audit/v2", + "experiment_id": "slm299-edit-reachability", + "seed_source": "root = Stack([], \"column\")", + "mode": "extended", + "max_edits": 8, + "node_budget": 15, + "generated_at": "2026-07-26T19:54:14Z", + "verdict_policy": "reachable_fraction is computed over decided cases only; UNKNOWN_BUDGET cases are reported separately and are never counted as unreachable; suites without a corpus are corpus_unavailable, never zero-reachable. Reachability is a space-coverage proof, never a model-quality claim.", + "suites": { + "train": { + "status": "corpus_unavailable", + "n_cases": 0, + "reachable_fraction": null + }, + "smoke": { + "status": "ok", + "n_cases": 3, + "n_decided": 3, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 3 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "smoke_hero_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_button_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "smoke_callout_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "held_out": { + "status": "ok", + "n_cases": 5, + "n_decided": 5, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 5 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "held_out_form_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_input_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_tabs_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "held_out_settings_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "adversarial": { + "status": "ok", + "n_cases": 4, + "n_decided": 3, + "n_unknown_budget": 1, + "n_reachable": 1, + "reachable_fraction": 0.333333, + "reason_histogram": { + "budget": 1, + "needs_direction_change": 1, + "reached": 1, + "unsupported_component": 1 + }, + "action_coverage": { + "ADD": 1, + "SET_PROPERTY": 1 + }, + "component_coverage": { + "TextContent": 1 + }, + "edit_lower_bound": { + "min": 2, + "median": 2, + "max": 2, + "mean": 2 + }, + "cases": [ + { + "id": "adv_empty_prompt_01", + "verdict": "PROVEN_REACHABLE", + "reason_code": "reached", + "edit_lower_bound": 2, + "path_length": 2 + }, + { + "id": "adv_dual_card_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_deep_nest_01", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "adv_many_buttons_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "ood": { + "status": "ok", + "n_cases": 4, + "n_decided": 4, + "n_unknown_budget": 0, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "unsupported_component": 4 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "ood_dashboard_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_gallery_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_modal_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "ood_auth_01", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "unsupported_component", + "edit_lower_bound": null, + "path_length": 0 + } + ] + }, + "rico": { + "status": "ok", + "n_cases": 35, + "n_decided": 1, + "n_unknown_budget": 34, + "n_reachable": 0, + "reachable_fraction": 0.0, + "reason_histogram": { + "budget": 34, + "needs_direction_change": 1 + }, + "action_coverage": {}, + "component_coverage": {}, + "edit_lower_bound": null, + "cases": [ + { + "id": "rico_eval_test_0", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_1", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_2", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_4", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_8", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_9", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_12", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_17", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_20", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_25", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_34", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_35", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_38", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_40", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_41", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_42", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_47", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_48", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_51", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_53", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_55", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_56", + "verdict": "PROVEN_UNREACHABLE", + "reason_code": "needs_direction_change", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_57", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_58", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_59", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_60", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_68", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_69", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_77", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_81", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_91", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_95", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_97", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_99", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + }, + { + "id": "rico_eval_test_104", + "verdict": "UNKNOWN_BUDGET", + "reason_code": "budget", + "edit_lower_bound": null, + "path_length": 0 + } + ] + } + }, + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "74e6cca3f84cae21bf2a4b11055f1c9495f144c9", + "code_dirty": true, + "components": { + "harness.experiments.slm299_edit_reachability": "v6" + }, + "stamped_at": "2026-07-26T19:54:53.642883+00:00" + }, + "x22_evidence_annotations": [ + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "adversarial", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.333333 over 3 decided adversarial cases (1 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on adversarial in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "held_out", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 5 decided held_out cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on held_out in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "ood", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 4 decided ood cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on ood in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "rico", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 1 decided rico cases (34 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on rico in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-x22-d3-kapur-tree-edit-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + }, + { + "date": "2026-07-26T19:54:14Z", + "target_doc": "docs/design/iter-efs0-04-x22-reproduction-20260717.md", + "suite": "smoke", + "annotation": "SLM-299 edit-reachability audit: from the standard X22 minimal seed, reachable_fraction=0.0 over 3 decided smoke cases (0 UNKNOWN_BUDGET, never counted unreachable). Suite-level quality readings of the X22 tree-edit decode on smoke in this document are bounded by that fraction: the unreachable share of gold programs cannot be produced by the decode space at all, so measured quality on those cases reflects space coverage, not model quality." + } + ] + }, + "comparison_note": "The SLM-305 baseline (iter-slm305-edit-language-20260724.md) ran at node_budget=120 against OLDER corpora (rico had 6 records, adversarial had no adv_deep_nest_01 UNKNOWN_BUDGET case, train's corpus existed as a committed fixture) -- it is not a same-budget, same-corpus baseline. VAR1-01's Arm A_baseline (docs/design/var1-01-set-property-probe-20260725.json) reproduces the PRE-VAR1-02 production action set at node_budget=15 against the SAME current corpora this run used, so it is the honest apples-to-apples 'before' comparison; VAR1-01's Arm B (set_property_action() what-if, same budget/corpus) is the probe's prediction this issue is gated on.", + "suite_summary": { + "train": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": "corpus_unavailable", + "var1_01_arm_b_set_property_what_if_node_budget_15": "corpus_unavailable", + "var1_02_production_node_budget_15": "corpus_unavailable", + "gate_production_at_or_below_arm_b": "n/a (corpus_unavailable)" + }, + "smoke": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": 0.0, + "var1_01_arm_b_set_property_what_if_node_budget_15": 0.0, + "var1_02_production_node_budget_15": 0.0, + "gate_production_at_or_below_arm_b": "ok (0.0 <= 0.0)" + }, + "held_out": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": 0.0, + "var1_01_arm_b_set_property_what_if_node_budget_15": 0.0, + "var1_02_production_node_budget_15": 0.0, + "gate_production_at_or_below_arm_b": "ok (0.0 <= 0.0)" + }, + "adversarial": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": 0.0, + "var1_01_arm_b_set_property_what_if_node_budget_15": 0.5, + "var1_02_production_node_budget_15": 0.333333, + "gate_production_at_or_below_arm_b": "ok (0.333333 <= 0.5)" + }, + "ood": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": 0.0, + "var1_01_arm_b_set_property_what_if_node_budget_15": 0.0, + "var1_02_production_node_budget_15": 0.0, + "gate_production_at_or_below_arm_b": "ok (0.0 <= 0.0)" + }, + "rico": { + "slm305_baseline_node_budget_120_older_corpora": 0.0, + "var1_01_arm_a_before_node_budget_15": 0.0, + "var1_01_arm_b_set_property_what_if_node_budget_15": "no_decided_cases", + "var1_02_production_node_budget_15": 0.0, + "gate_production_at_or_below_arm_b": "ok (arm B had no decided cases to exceed)" + } + }, + "verdict_flips_vs_var1_01_arm_a": { + "smoke": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "held_out": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "adversarial": { + "n_verdict_flips": 1, + "verdict_flips": [ + { + "id": "adv_empty_prompt_01", + "v1": "PROVEN_UNREACHABLE", + "extended": "PROVEN_REACHABLE" + } + ], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.333333, + "action_cost_delta": null + }, + "ood": { + "n_verdict_flips": 0, + "verdict_flips": [], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + }, + "rico": { + "n_verdict_flips": 34, + "verdict_flips": [ + { + "id": "rico_eval_test_0", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_1", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_104", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_12", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_17", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_2", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_20", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_25", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_34", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_35", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_38", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_4", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_40", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_41", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_42", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_47", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_48", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_51", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_53", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_55", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_57", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_58", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_59", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_60", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_68", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_69", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_77", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_8", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_81", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_9", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_91", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_95", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_97", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + }, + { + "id": "rico_eval_test_99", + "v1": "PROVEN_UNREACHABLE", + "extended": "UNKNOWN_BUDGET" + } + ], + "reachable_fraction_v1": 0.0, + "reachable_fraction_extended": 0.0, + "action_cost_delta": null + } + }, + "honest_verdict": { + "claim_class": "capability", + "statement": "This is a space-coverage (reachability) delta only. Adding ACTION_SET_PROPERTY closes exactly the gap VAR1-01 (SLM-424) confirmed: adv_empty_prompt_01 flips PROVEN_UNREACHABLE -> PROVEN_REACHABLE (edit_lower_bound=2, path=[ADD, SET_PROPERTY]), matching VAR1-01 Arm B's confirmed flip exactly (same case, same path shape, same 2-edit bound).", + "not_claimed": [ + "ship readiness", + "promotion of any checkpoint", + "champion-model change", + "improved output quality (reachability is space coverage, not a model-quality claim; no training, evaluation, or promotion ran as part of this issue)" + ], + "discrepancy_disclosed": "rico's reachable_fraction did NOT improve (stays 0.0) and its DECIDED case count dropped sharply: 35/35 decided under VAR1-01's Arm A (all PROVEN_UNREACHABLE, needs_direction_change) versus only 1/35 decided in this production run (34 became UNKNOWN_BUDGET). This is not a regression in reachability -- it is the loosening of an invariant that was previously (and, before this change, correctly) too strict: most rico roots omit a direction arg entirely (`Stack([...])`, rest=\"\"), which the pack's own container_rests domain already permits, so the old hard root-rest-must-equal-seed check no longer holds once SET_PROPERTY can legitimately rewrite it -- but confirming the REST of each transformation (adding several Card+leaf subtrees) is a deeper search than node_budget=15/max_edits=8 can decide. Per AGENTS.md I14, UNKNOWN_BUDGET is inconclusive, never evidence of unreachability, and is reported separately here rather than folded into a falsely-precise 0.0. A longer, separately-run job at a larger node_budget (outside this repository's MAX_RUN_MINUTES=3 cap for this session) is needed to resolve those 34 cases either way; this is an honest known gap, not a result picked to look favorable.", + "gate_check": "Per suite, the production reachable_fraction lands at or below VAR1-01 Arm B's (see suite_summary.*.gate_production_at_or_below_arm_b): adversarial is 0.333333 <= 0.5 (Arm B's BFS treated 2 cases as UNKNOWN_BUDGET where production's more precise, pack-domain-bounded invariant decisively proved one of them (adv_many_buttons_01, whose target needs a \"row\" rest the pack does not declare) PROVEN_UNREACHABLE instead of leaving it inconclusive -- a strictly MORE decisive, not less correct, result). No suite landed above its Arm B counterpart; no red flag." + }, + "out_of_scope": [ + "Training, evaluating, promoting, or syncing any checkpoint.", + "Widening the component inventory (VAR0-03/SLM-426, already done).", + "Any claim that improved reachability implies improved output quality." + ], + "version_stamp": { + "stamp_schema": "version_stamp/v1", + "code_commit": "74e6cca3f84cae21bf2a4b11055f1c9495f144c9", + "code_dirty": true, + "components": { + "harness.experiments.slm299_edit_reachability": "v6" + }, + "stamped_at": "2026-07-26T19:57:53.504388+00:00" + } +} diff --git a/docs/design/var1-02-set-property-action-20260726.md b/docs/design/var1-02-set-property-action-20260726.md new file mode 100644 index 000000000..45e7a72be --- /dev/null +++ b/docs/design/var1-02-set-property-action-20260726.md @@ -0,0 +1,160 @@ +# VAR1-02 (SLM-425): SET_PROPERTY action -- production reachability delta + +- generated_at: `2026-07-26T19:54:14Z` +- gated on: `docs/design/var1-01-set-property-probe-20260725.md` (SLM-424/VAR1-01) +- hypothetical: `false` -- this is the REAL, production action space, checkpoint + format, and analyzer, not a what-if probe. +- mode: `extended`, max_edits: 8, node_budget: 15 (see "Why node_budget=15" below) +- seed: `root = Stack([], "column")` + +> Reachability is space coverage, not model quality: no quality claim follows +> from these proofs alone. No training, evaluation, or promotion ran as part +> of this issue. + +## What changed + +`ACTION_SET_PROPERTY` (index 11; `N_ACTIONS` 11 -> 12) rebinds an existing +container's declared `rest` (its enum/direction argument) to another value of +the SAME pack-declared domain (`DslPack.component_property_domains`) -- root +included, since (unlike `REMOVE_CONTAINER`) it never removes or re-mints a +node. `CHECKPOINT_FORMAT` bumped 2 -> 3; a format-2 checkpoint fails closed on +load pointing at `checkpoint_migrate.migrate_tree_edit_checkpoint`, which +already upgrades any `format_version < CHECKPOINT_FORMAT` generically (no new +function needed -- see the code changes list below). The tree-edit variant's +`VariantContractV1.kernel_ops` now declares `openui.set_property`. The SLM-299 +analyzer's `_check_invariants` now recognizes the real action (pack-domain +bounded, forced on in extended mode) separately from VAR1-01's hypothetical +`set_property_action()` (which may use a wider what-if domain and still +defers to BFS exactly as before) -- every VAR1-01 probe test continues to pass +unchanged. + +## Reachable fraction by suite: four points of comparison + +| suite | SLM-305 baseline (node_budget=120, older corpora) | VAR1-01 Arm A (before, node_budget=15, current corpora) | VAR1-01 Arm B (`set_property_action()` what-if, node_budget=15) | VAR1-02 production (this run, node_budget=15) | gate: production <= Arm B | +| --- | --- | --- | --- | --- | --- | +| train | 0.0 | corpus_unavailable | corpus_unavailable | corpus_unavailable | n/a | +| smoke | 0.0 | 0.0 | 0.0 | 0.0 | ok (0.0 <= 0.0) | +| held_out | 0.0 | 0.0 | 0.0 | 0.0 | ok (0.0 <= 0.0) | +| adversarial | 0.0 | 0.0 | 0.5 | 0.333333 | ok (0.333333 <= 0.5) | +| ood | 0.0 | 0.0 | 0.0 | 0.0 | ok (0.0 <= 0.0) | +| rico | 0.0 | 0.0 | no_decided_cases | 0.0 | ok (Arm B had no decided cases to exceed) | + +The SLM-305 baseline (`docs/design/iter-slm305-edit-language-20260724.md`) is +**not** a same-budget, same-corpus comparison: it ran at `node_budget=120` +against older corpora (rico had 6 records, not 35; adversarial had no +`adv_deep_nest_01` UNKNOWN_BUDGET case; `train`'s corpus existed as a +committed fixture, now a gitignored generated artifact) -- see VAR1-01's own +note on this. VAR1-01's **Arm A_baseline** reproduces the pre-VAR1-02 +production action set at `node_budget=15` against the exact same corpora this +run used, so it is the honest apples-to-apples "before"; **Arm B** is the +probe's what-if prediction this issue is gated on. + +### Why node_budget=15, not the issue's suggested 120 + +A single-arm run at `node_budget=120` was attempted first and did not finish +inside a 170s wall-clock cap (`MAX_RUN_MINUTES=3`); a timed-out run is never +evidence per AGENTS.md. `node_budget=15` (VAR1-01's own precedent) completed +in 41.6s. This is a materially weaker probe on the `rico` suite in particular +(see the discrepancy note below); a longer, separately-run job at a larger +budget outside this session's cap would be needed to fully resolve it. + +## Per-suite verdict flips vs VAR1-01 Arm A (the same-corpus "before") + +- **smoke / held_out / ood**: no flips (0 each) -- these suites' gaps are all + `unsupported_component` (component-inventory gap, out of this issue's + scope; VAR0-03/SLM-426 already closed the analogous inventory gap + elsewhere). +- **adversarial**: 1 flip. + - `adv_empty_prompt_01`: `PROVEN_UNREACHABLE` (`needs_direction_change`) -> + `PROVEN_REACHABLE` (`reached`, `edit_lower_bound=2`, + `path=[ADD, SET_PROPERTY]`) -- **CONFIRMED**, and matches VAR1-01 Arm B's + confirmed flip on this exact case, same path shape, same 2-edit bound. + - `adv_many_buttons_01` stays `PROVEN_UNREACHABLE` (`needs_direction_change`) + -- its target needs a `"row"` rest, which the pack does not declare in + `component_property_domains` (only `', "column"'` and `""`), so + `ACTION_SET_PROPERTY` genuinely cannot produce it. Production's + pack-domain-bounded invariant proves this case decisively; VAR1-01 Arm B + (unconditional invariant skip, unbounded what-if domain) left it as + `UNKNOWN_BUDGET` instead. **More decisive, not less correct.** + - `adv_deep_nest_01` stays `UNKNOWN_BUDGET` in both (budget-bound, + unaffected by this change). +- **rico**: 34 flips, all `PROVEN_UNREACHABLE` (`needs_direction_change`) -> + `UNKNOWN_BUDGET` (`budget`). See the discrepancy note below -- this is an + honest, disclosed side effect, not a reachability regression. + +## Honest-verdict + +- **Claim class: `capability`** (space coverage only). Not claimed: ship + readiness, promotion of any checkpoint, a champion-model change, or that + improved reachability implies improved output quality. No training, + evaluation, or promotion ran as part of this issue. +- **What is confirmed:** `adv_empty_prompt_01` flips to `PROVEN_REACHABLE`, + reproducing VAR1-01 Arm B's confirmed positive exactly (same case, same + 2-edit path shape: `ADD` then `SET_PROPERTY`). +- **Gate check (issue requirement: production <= Arm B per suite, else red + flag):** every suite lands at or below its Arm B counterpart (see the table + above) -- **no red flag.** `adversarial`'s 0.333333 <= 0.5 is the only + suite with a numeric Arm B value below 1.0; the apparent "regression" from + 0.5 to 0.333333 is explained by production **deciding** one more case + (`adv_many_buttons_01`, proven genuinely unreachable) than Arm B's cruder + unconditional-skip invariant could -- a strictly more decisive result, not + a worse one. +- **Discrepancy disclosed honestly (not picked for looking favorable):** + `rico`'s reachable_fraction did **not** improve (stays `0.0`), and its + decided-case count collapsed: VAR1-01 Arm A decided all 35/35 cases + (`PROVEN_UNREACHABLE`, `needs_direction_change`); this production run + decides only 1/35 (`rico_eval_test_56`, whose target needs a `"row"` rest + outside the pack's domain -- still decisively unreachable), and the other + 34 become `UNKNOWN_BUDGET`. This is **not** a reachability regression: most + `rico` targets omit a direction argument entirely (`root = + Stack([...])`, rest `""`), which the pack's own `container_rests` domain + already permits -- so the OLD hard "root rest must equal seed's" check was + never actually sound once a real rest-mutation action exists; it was + masking the true (deeper, structural) reachability question behind a + cheap-but-now-recognized-as-overly-strict proof. Confirming or refuting the + REST of each transformation (typically several `ADD_CONTAINER`/ + `INSERT_SUBTREE`/`BIND_PLACEHOLDER` steps to rebuild multiple + `Card`+leaf subtrees) needs a deeper search than `node_budget=15, + max_edits=8` can decide. Per AGENTS.md I14, `UNKNOWN_BUDGET` is + inconclusive and never evidence of unreachability; it is reported + separately here, not folded into a falsely-precise `0.0`. A longer, + separately-run job at a larger `node_budget` (outside this session's + `MAX_RUN_MINUTES=3` cap) is required to resolve those 34 cases either way. + +## Files changed + +- `src/slm_training/models/tree_edit_diffusion.py` -- `ACTION_SET_PROPERTY`, + `N_ACTIONS=12`, `CHECKPOINT_FORMAT=3`, `EditDomain.component_property_domains` + / `.property_names`, `TreeEditSpace.apply` branch (root-inclusive, real + parser re-validated), `sample_mutation` inverse-edit generation, + `_enumerate_edits` decode-time scoring. +- `src/slm_training/models/checkpoint_migrate.py` -- docstring only; + `migrate_tree_edit_checkpoint` already upgrades any + `source_format < CHECKPOINT_FORMAT` generically (shape-driven, not + hardcoded to format 1), so format 2 -> 3 needed no new code path. +- `src/slm_training/harnesses/experiments/slm299_edit_reachability.py` -- + `_enumerate_children` gains the real `ACTION_SET_PROPERTY` transitions; + `_check_invariants` distinguishes the real (pack-domain-bounded) capability + from VAR1-01's hypothetical (potentially wider-domain) one via + `hypothetical_set_property_domain`; `analyze_reachability` forces + `set_property` on in extended mode alongside the existing `container_add` + forcing. +- `src/slm_training/dsl/variants.py` -- `tree_edit_diffusion` variant's + `kernel_ops` now includes `openui.set_property`; regenerated + `src/slm_training/resources/variant_registry.json`. +- `tests/test_models/test_tree_edit_diffusion.py` -- new tests: apply/inverse + round-trip on root and a non-root container, illegal-value rejection via + the real parser (not just an index bounds-check), format-2 fail-closed + load, and format-2->3 migration with bit-identical logits on the + pre-existing 11 action rows. +- `src/slm_training/resources/versions.json` -- bumped + `harness.experiments.slm299_edit_reachability` (v5->v6), `dsl.variants` + (v2->v3), `model.twotower` (v255->v256, watches + `checkpoint_migrate.py`). +- This doc pair. + +## Out of scope (per issue) + +- Training, evaluating, promoting, or syncing any checkpoint. +- Widening the component inventory (VAR0-03/SLM-426, already done). +- Any claim that improved reachability implies improved output quality. diff --git a/src/slm_training/dsl/variants.py b/src/slm_training/dsl/variants.py index 449f18fc2..833db7d73 100644 --- a/src/slm_training/dsl/variants.py +++ b/src/slm_training/dsl/variants.py @@ -247,7 +247,8 @@ def build_variant_contracts() -> tuple[VariantContractV1, ...]: pack_id="openui", action_alphabet_id="tree_edit_diffusion.edit_actions", action_alphabet_fingerprint=_alphabet_fingerprint(_tree_edit_action_names()), - kernel_ops=(), + # SLM-425 (VAR1-02): ACTION_SET_PROPERTY realizes this kernel op. + kernel_ops=("openui.set_property",), seed_policy_id="tree_edit_diffusion.minimal_valid_program_seed", inventory_source="pack", source_path=_TREE_EDIT_DIFFUSION_SOURCE, diff --git a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py index 91e646fa1..8f0697873 100644 --- a/src/slm_training/harnesses/experiments/slm299_edit_reachability.py +++ b/src/slm_training/harnesses/experiments/slm299_edit_reachability.py @@ -42,6 +42,7 @@ ACTION_REPLACE, ACTION_REPLACE_STATEMENT, ACTION_REPLACE_SUBTREE, + ACTION_SET_PROPERTY, MAX_SLOTS, Edit, Statement, @@ -179,10 +180,14 @@ def set_property_action( ) -> ExtraAction: """VAR1-01 hypothetical: rebind an existing container's ``rest`` (its enum/direction argument) in place, without minting or removing any - statement. No real action in the deployed edit space edits ``rest`` on an - existing node — see ``_check_invariants``'s ``needs_direction_change`` - reason. This is a *what-if* probe of a property-mutation action class, - never a production action; see VAR1-01 (SLM-424). + statement, drawing from ``rests`` — which may be WIDER than the real + ``TreeEditSpace.container_rests`` domain (the default includes ``"row"``, + which the pack does not declare). SLM-425 (VAR1-02) added a REAL + ``ACTION_SET_PROPERTY`` gated on this probe's confirmed result, but its + domain is exactly the pack-declared ``container_rests`` — narrower than + this what-if's default. This factory (and the capability it grants) stays + useful for exploring domains wider than the pack currently declares; see + VAR1-01 (SLM-424) and VAR1-02 (SLM-425). """ def _generate( @@ -375,6 +380,7 @@ def _check_invariants( capabilities: frozenset[str] = frozenset(), *, extended: bool = False, + hypothetical_set_property_domain: bool = False, ) -> str | None: """Structural impossibility proofs over the EXACT action set. @@ -386,7 +392,17 @@ def _check_invariants( leaves only to inventory slots, and the V0.5 statement actions mint only canonical templates — the invariant reasons fire only when the corresponding REAL - action of the analyzed mode is absent. + action of the analyzed mode is absent. SLM-425 (VAR1-02): the extended + space also has a REAL ``ACTION_SET_PROPERTY`` that rebinds any existing + container's (root included) ``rest`` to another value of + ``space.container_rests`` — the pack-declared domain, narrower than a + hypothetical ``set_property_action()``'s (default, wider) ``rests``. + ``hypothetical_set_property_domain`` distinguishes the two: it is True + only when the ``set_property`` capability comes from an *extra_action* + (VAR1-01 what-if, unbounded domain — the cheap proof below cannot rule + those out generically, so it is skipped entirely, as before); it is False + when the capability is the real, pack-domain-bounded action (forced on + in extended mode), where the cheap proof stays precise. """ seed_containers = [s for s in seed if s.has_list] target_containers = [s for s in target if s.has_list] @@ -431,28 +447,38 @@ def _check_invariants( if normalized not in inventory: return REASON_NEEDS_SLOT_REBIND - # REPLACE preserves the container's raw enum/direction arg text (rest); - # no real action edits it. Containers minted by the container-creating - # actions carry exactly one of CONTAINER_RESTS. Every target container - # rest must therefore be a seed rest or (with container_add) a candidate - # mint rest; without any container-creating action the multisets must - # match exactly. - if "set_property" in capabilities: - # A rest-mutation action can produce any target rest on any - # container, root included -- skip both rest checks below entirely. + # REPLACE preserves the container's raw enum/direction arg text (rest). + # Containers minted by the container-creating actions carry exactly one + # of CONTAINER_RESTS; SET_PROPERTY (SLM-425/VAR1-02) rewrites an + # existing container's rest to exactly one of CONTAINER_RESTS too, root + # included. Every target container rest must therefore be a seed rest or + # a candidate mint/set rest; without either capability the multisets + # must match exactly. + if "set_property" in capabilities and hypothetical_set_property_domain: + # VAR1-01 what-if only: an ExtraAction may draw from ANY domain (its + # own ``rests``, unbounded by the pack), so no cheap proof is + # derivable generically here -- defer entirely to BFS (legacy + # behavior, unchanged since VAR1-01). return None seed_rests = sorted(s.rest for s in seed_containers) target_rests = sorted(s.rest for s in target_containers) - # Root can never be removed or re-minted, and REPLACE preserves rest: - # the target root's rest must equal the seed root's rest in every mode. seed_root_rest = next( (s.rest for s in seed_containers if s.name == "root"), None ) - for stmt in target_containers: - if stmt.name == "root" and stmt.rest != seed_root_rest: - return REASON_NEEDS_DIRECTION_CHANGE - if "container_add" in capabilities: + if "set_property" not in capabilities: + # No action edits an existing container's rest in place: the root + # can never be removed or re-minted, and REPLACE preserves rest, so + # the target root's rest must equal the seed root's rest. + for stmt in target_containers: + if stmt.name == "root" and stmt.rest != seed_root_rest: + return REASON_NEEDS_DIRECTION_CHANGE + if "container_add" in capabilities or "set_property" in capabilities: + # Either a fresh container can be minted with any of + # space.container_rests, or an existing one's rest (root included) + # can be rewritten to any of space.container_rests -- so every + # target rest must be drawn from the seed's own rests or that real + # domain. allowed = set(seed_rests) | set(space.container_rests) if any(rest not in allowed for rest in target_rests): return REASON_NEEDS_DIRECTION_CHANGE @@ -599,6 +625,23 @@ def _replace_subtree_ok(stmt: Statement) -> bool: }, ) ) + # SLM-425 (VAR1-02): rebind this container's rest in place (root + # included) to another value of the same pack-declared domain -- + # the property-mutation action class VAR1-01 proved missing. + for rest_idx in range(len(space.container_rests)): + edit = Edit(ACTION_SET_PROPERTY, stmt_idx, target=rest_idx) + nxt = space.apply(statements, edit, inventory, pre) + if nxt is not None: + children.append( + ( + nxt, + { + "action": "SET_PROPERTY", + "stmt": stmt_idx, + "rest": space.container_rests[rest_idx], + }, + ) + ) else: for slot_idx in range(n_slots): edit = Edit(ACTION_BIND_PLACEHOLDER, stmt_idx, slot=slot_idx) @@ -714,16 +757,27 @@ def analyze_reachability( details=details, ) - capabilities = frozenset().union( + extra_capabilities = frozenset().union( *(a.capabilities for a in extra_actions) ) if extra_actions else frozenset() + capabilities = extra_capabilities if mode == "extended": # ADD_CONTAINER / INSERT_SUBTREE are REAL actions in the extended # space, so the container_add invariant must not fire; the synthetic # add_container_action what-if lane is retired for this space. - capabilities = capabilities | frozenset({"container_add"}) + # SET_PROPERTY (SLM-425/VAR1-02) is likewise REAL in the extended + # space (pack-domain-bounded); see ``_check_invariants``'s + # ``hypothetical_set_property_domain`` for how that differs from an + # extra_action's (potentially wider) what-if domain. + capabilities = capabilities | frozenset({"container_add", "set_property"}) fired = _check_invariants( - target, seed, inventory, space, capabilities, extended=(mode == "extended") + target, + seed, + inventory, + space, + capabilities, + extended=(mode == "extended"), + hypothetical_set_property_domain="set_property" in extra_capabilities, ) if fired is not None: return ReachabilityCase( diff --git a/src/slm_training/models/checkpoint_migrate.py b/src/slm_training/models/checkpoint_migrate.py index 46e738705..6f50ec8ae 100644 --- a/src/slm_training/models/checkpoint_migrate.py +++ b/src/slm_training/models/checkpoint_migrate.py @@ -301,13 +301,18 @@ def migrate_tree_edit_checkpoint( output_checkpoint: Path | str, device: str = "cpu", ) -> dict: - """Warm-start a tree-edit-diffusion format-1 checkpoint to format 2. - - SLM-305 grew ``action_head`` from 4 to ``N_ACTIONS`` rows (extended edit - language). All same-shape tensors are copied verbatim; the old action-head - rows are preserved in the first rows of the new head and the new action - rows stay randomly initialized. A ``.migrate.json`` report is written next - to the output checkpoint. + """Warm-start an older tree-edit-diffusion checkpoint to the current format. + + Format-agnostic: it upgrades any checkpoint whose + ``format_version < TreeEditDiffusionModel.CHECKPOINT_FORMAT`` (format 1 + or format 2 today) to the live format by shape, not by a hardcoded source + version. SLM-305 grew ``action_head`` from 4 to 11 rows (extended edit + language, format 1 -> 2); SLM-425 (VAR1-02) grew it again from 11 to 12 + rows for ``ACTION_SET_PROPERTY`` (format 2 -> 3). All same-shape tensors + are copied verbatim; the old action-head rows are preserved in the first + rows of the new head (bit-identical logits on those rows) and the new + action row(s) stay randomly initialized. A ``.migrate.json`` report is + written next to the output checkpoint. """ from slm_training.models.tree_edit_diffusion import ( TreeEditDiffusionConfig, diff --git a/src/slm_training/models/tree_edit_diffusion.py b/src/slm_training/models/tree_edit_diffusion.py index b34ec2e5c..81934bb48 100644 --- a/src/slm_training/models/tree_edit_diffusion.py +++ b/src/slm_training/models/tree_edit_diffusion.py @@ -22,10 +22,10 @@ import json import random import re -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, field from functools import lru_cache from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Mapping import torch import torch.nn as nn @@ -62,7 +62,15 @@ ACTION_INSERT_STATEMENT = 8 # insert a canonical V0.5 state/query/mutation statement ACTION_REPLACE_STATEMENT = 9 # swap one canonical V0.5 statement for another ACTION_BIND_PLACEHOLDER = 10 # (re)bind a leaf's slot to an inventory placeholder -N_ACTIONS = 11 +# SLM-425 (VAR1-02): property mutation on an EXISTING node -- orthogonal to +# every action above (all structural: add/remove/replace a node or subtree). +# VAR1-01 (SLM-424) proved this the missing action class behind +# reachable_fraction=0.0 via a hypothetical probe; this is the real action, +# gated on that probe's confirmed result. Domain is pack-provided (see +# ``EditDomain.component_property_domains`` / ``.property_names`` below), not +# a module constant. +ACTION_SET_PROPERTY = 11 # rebind an existing container's declared property +N_ACTIONS = 12 MAX_STMTS = 24 MAX_SLOTS = 16 @@ -75,6 +83,14 @@ class EditDomain: container_components: tuple[str, ...] container_rests: tuple[str, ...] statement_templates: tuple[tuple[str, str], ...] + # SLM-425 (VAR1-02): the full per-component property->value-domain map + # (currently just each container's "rest"/direction arg) plus the sorted + # property-name alphabet ``ACTION_SET_PROPERTY.comp`` indexes into. Both + # are pack-derived, never module constants. + component_property_domains: Mapping[str, Mapping[str, tuple[str, ...]]] = field( + default_factory=dict + ) + property_names: tuple[str, ...] = () def edit_domain(pack_id: str = "openui") -> EditDomain: @@ -87,11 +103,24 @@ def edit_domain(pack_id: str = "openui") -> EditDomain: ) if not pack.leaf_components or not pack.container_components or not rests: raise ValueError(f"pack {pack_id!r} does not provide a tree-edit domain") + property_names = tuple( + sorted( + { + prop + for values in pack.component_property_domains.values() + for prop in values + } + ) + ) return EditDomain( leaf_components=tuple(pack.leaf_components), container_components=tuple(pack.container_components), container_rests=tuple(dict.fromkeys(rests)), statement_templates=tuple(pack.statement_templates), + component_property_domains={ + comp: dict(values) for comp, values in pack.component_property_domains.items() + }, + property_names=property_names, ) @@ -273,6 +302,8 @@ def __init__( self.container_components = self.domain.container_components self.container_rests = self.domain.container_rests self.statement_templates = self.domain.statement_templates + self.component_property_domains = self.domain.component_property_domains + self.property_names = self.domain.property_names def fresh_name(self, statements: list[Statement]) -> str: taken = {s.name for s in statements} @@ -545,6 +576,46 @@ def apply( return None placeholder = self._placeholder(inventory, edit.slot) target.rest = json.dumps(placeholder, ensure_ascii=False) + elif edit.action == ACTION_SET_PROPERTY: + # SLM-425 (VAR1-02): rebind an EXISTING container's declared + # property (currently the "rest" enum/direction arg) to another + # value from the SAME pack-declared domain for that component. No + # statement is minted or removed -- this is the orthogonal + # property-mutation action class VAR1-01 proved missing + # (structural edits alone cannot flip a container's own rest, + # including the root's -- see needs_direction_change in + # slm299_edit_reachability). Preconditions: target is a + # container -- root included. Unlike REMOVE_CONTAINER (which can + # never touch root because removal would have to re-mint it), + # SET_PROPERTY only ever rewrites an existing field in place, so + # the root guard used by the structural actions does not apply + # here; edit.comp indexes a known property name, edit.target + # indexes a legal value for THIS component's declared domain, and + # the new value must be a real change. Rebuilt via the + # structured ``Statement`` fields -- never regex/text surgery on + # program text (see the V0.5 construction note above) -- and the + # common tail below re-validates the rebuilt statement through + # the real parser. Inverse: SET_PROPERTY restoring the prior + # value's index -- always expressible (same domain both ways, + # root included). + if not (0 <= edit.stmt < len(working)): + return None + target = working[edit.stmt] + if not target.has_list: + return None + domain = self.component_property_domains.get(target.comp) + if not domain or not (0 <= edit.comp < len(self.property_names)): + return None + prop = self.property_names[edit.comp] + values = domain.get(prop) + if prop != "rest" or not values or not (0 <= edit.target < len(values)): + return None + new_value = values[edit.target] + if new_value == target.rest: + return None + working[edit.stmt] = Statement( + target.name, target.comp, list(target.children), new_value, target.has_list + ) else: return None if pre_validate is not None and not pre_validate(working): @@ -575,6 +646,7 @@ def sample_mutation( ACTION_INSERT_STATEMENT, ACTION_REPLACE_STATEMENT, ACTION_BIND_PLACEHOLDER, + ACTION_SET_PROPERTY, ) ) if kind == ACTION_REPLACE: @@ -817,6 +889,34 @@ def sample_mutation( continue inverse = Edit(ACTION_BIND_PLACEHOLDER, idx, slot=old_slot) return mutated, inverse + if kind == ACTION_SET_PROPERTY: + # Mutation = rebind an existing container's "rest" (root + # included -- see the guard note on ACTION_SET_PROPERTY in + # ``apply``); inverse = SET_PROPERTY restoring the old value. + containers = [i for i, s in enumerate(statements) if s.has_list] + if not containers: + continue + idx = rng.choice(containers) + stmt = statements[idx] + domain = self.component_property_domains.get(stmt.comp) + if not domain or "rest" not in self.property_names: + continue + values = domain.get("rest") + if not values or stmt.rest not in values: + continue + prop_idx = self.property_names.index("rest") + old_idx = values.index(stmt.rest) + choices = [i for i in range(len(values)) if i != old_idx] + if not choices: + continue + mutation = Edit( + ACTION_SET_PROPERTY, idx, prop_idx, target=rng.choice(choices) + ) + mutated = self.apply(statements, mutation, inventory) + if mutated is None: + continue + inverse = Edit(ACTION_SET_PROPERTY, idx, prop_idx, target=old_idx) + return mutated, inverse # Mutation = remove a leaf; inverse = ADD it back. removable = [ i @@ -951,9 +1051,12 @@ class TreeEditDiffusionModel(nn.Module): """Prompt-conditioned Kapur-style edit policy + value search (X22).""" # Format 2 (SLM-305): action_head grew to N_ACTIONS=11 with the extended - # edit language. Format-1 checkpoints fail closed here; warm-start them - # via ``checkpoint_migrate.migrate_tree_edit_checkpoint``. - CHECKPOINT_FORMAT = 2 + # edit language. Format 3 (SLM-425/VAR1-02): action_head grew again to + # N_ACTIONS=12 for ACTION_SET_PROPERTY (property mutation on an existing + # node). Format-1 and format-2 checkpoints both fail closed here; warm- + # start them via ``checkpoint_migrate.migrate_tree_edit_checkpoint`` + # (format-agnostic: it upgrades any ``source_format < CHECKPOINT_FORMAT``). + CHECKPOINT_FORMAT = 3 def __init__( self, @@ -1260,6 +1363,13 @@ def _enumerate_edits( Edit(ACTION_REMOVE_CONTAINER, stmt), ) ) + for rest_idx in range(len(CONTAINER_RESTS)): + scored.append( + ( + float(action_lp[ACTION_SET_PROPERTY]) + base, + Edit(ACTION_SET_PROPERTY, stmt, target=rest_idx), + ) + ) for payload in range(len(V05_TEMPLATES)): scored.append( ( diff --git a/src/slm_training/resources/variant_registry.json b/src/slm_training/resources/variant_registry.json index 1ea7a0b0b..3b61a7a91 100644 --- a/src/slm_training/resources/variant_registry.json +++ b/src/slm_training/resources/variant_registry.json @@ -1,6 +1,6 @@ { "count": 3, - "fingerprint": "2a1a10b9b397ff07ec4eb8330c8b57c3589a7d23087ea676ca74e836f9aa544a", + "fingerprint": "cd09cb02571e27c1cd6cf952a3240ec9ee835110213c99ad2cbd45ebdb209209", "schema": "variant_contract/v1", "variants": [ { @@ -15,10 +15,12 @@ "variant_id": "repl_operators" }, { - "action_alphabet_fingerprint": "ab2662a497d8359ffaee46ebbd4bee3789f5b0f2accaf8bf46c5dee489622dab", + "action_alphabet_fingerprint": "1de83afe8bc7db333bf0cac040a19fa355c34d28653a508343e45f136a1dac4c", "action_alphabet_id": "tree_edit_diffusion.edit_actions", "inventory_source": "pack", - "kernel_ops": [], + "kernel_ops": [ + "openui.set_property" + ], "pack_id": "openui", "schema": "variant_contract/v1", "seed_policy_id": "tree_edit_diffusion.minimal_valid_program_seed", diff --git a/src/slm_training/resources/versions.json b/src/slm_training/resources/versions.json index c1687677c..fd8662933 100644 --- a/src/slm_training/resources/versions.json +++ b/src/slm_training/resources/versions.json @@ -5275,7 +5275,7 @@ ] }, "harness.experiments.slm299_edit_reachability": { - "version": "v5", + "version": "v6", "kind": "harness", "paths": [ "src/slm_training/harnesses/experiments/slm299_edit_reachability.py", @@ -5288,9 +5288,16 @@ "docs/design/iter-slm305-edit-language-20260724.md", "tests/test_models/test_tree_edit_diffusion.py", "docs/design/var0-03-pack-derived-inventory-20260726.json", - "docs/design/var0-03-pack-derived-inventory-20260726.md" + "docs/design/var0-03-pack-derived-inventory-20260726.md", + "docs/design/var1-02-set-property-action-20260726.json", + "docs/design/var1-02-set-property-action-20260726.md" ], "history": [ + { + "version": "v6", + "date": "2026-07-26", + "note": "SLM-425 (VAR1-02): add a REAL ACTION_SET_PROPERTY (N_ACTIONS 11->12, CHECKPOINT_FORMAT 2->3) gated on the VAR1-01 hypothetical probe's confirmed result -- rebinds an existing container's rest (root included) to another pack-declared container_rests value, with a safe self-inverse. _check_invariants' needs_direction_change proof now distinguishes the real, pack-domain-bounded action (precise cheap proof, forced on in extended mode) from an extra_action's potentially-wider hypothetical domain (VAR1-01's set_property_action(), which still defers to BFS exactly as before -- all existing VAR1-01 probe tests pass unchanged). Re-ran the production (non-hypothetical) SLM-299 audit; see docs/design/var1-02-set-property-action-20260726.{md,json} for the per-suite delta against the SLM-305 baseline and the VAR1-01 arm B probe." + }, { "version": "v5", "date": "2026-07-26", @@ -7469,7 +7476,7 @@ ] }, "model.twotower": { - "version": "v255", + "version": "v256", "kind": "harness", "paths": [ "src/slm_training/models/twotower.py", @@ -7511,6 +7518,11 @@ "src/slm_training/resources/decode/speculative_ngram_v1.json" ], "history": [ + { + "version": "v256", + "date": "2026-07-26", + "note": "SLM-425 (VAR1-02): migrate_tree_edit_checkpoint's docstring now documents the format-2->3 upgrade (N_ACTIONS 11->12 for ACTION_SET_PROPERTY) alongside the existing format-1->2 precedent; the function itself is already format-agnostic (upgrades any source_format < CHECKPOINT_FORMAT by shape) so no behavior changed." + }, { "version": "v255", "date": "2026-07-26", @@ -9300,7 +9312,7 @@ ] }, "dsl.variants": { - "version": "v2", + "version": "v3", "kind": "harness", "paths": [ "src/slm_training/dsl/variants.py", @@ -9308,6 +9320,11 @@ "tests/test_dsl/test_variants.py" ], "history": [ + { + "version": "v3", + "date": "2026-07-26", + "note": "SLM-425 (VAR1-02): tree_edit_diffusion's kernel_ops now declares openui.set_property (ACTION_SET_PROPERTY realizes this kernel op); regenerated the derived variant registry (new action_alphabet_fingerprint from the added ACTION_* constant plus the kernel_ops change)." + }, { "version": "v2", "date": "2026-07-26", diff --git a/tests/test_models/test_tree_edit_diffusion.py b/tests/test_models/test_tree_edit_diffusion.py index 02bc18498..46f04de40 100644 --- a/tests/test_models/test_tree_edit_diffusion.py +++ b/tests/test_models/test_tree_edit_diffusion.py @@ -66,6 +66,7 @@ def test_mutations_preserve_validity_and_inverse_restores() -> None: ACTION_REMOVE_CONTAINER, ACTION_REPLACE_STATEMENT, ACTION_REPLACE_SUBTREE, + ACTION_SET_PROPERTY, Edit, ) @@ -229,7 +230,7 @@ def test_checkpoint_format2_fail_closed_and_migration(tmp_path) -> None: source_checkpoint=old_path, output_checkpoint=out_path ) assert report["source_format_version"] == 1 - assert report["output_format_version"] == 2 + assert report["output_format_version"] == TreeEditDiffusionModel.CHECKPOINT_FORMAT assert report["preserved_action_head_rows"] == 4 assert (tmp_path / "ckpt_migrated.migrate.json").exists() migrated = TreeEditDiffusionModel.from_checkpoint(out_path, device="cpu") @@ -290,3 +291,190 @@ def test_training_loss_decode_all_valid_and_checkpoint(tmp_path) -> None: ] ) assert reproduced == outputs + + +# --- SLM-425 (VAR1-02): SET_PROPERTY action -------------------------------- + + +def test_set_property_apply_and_invert_root_and_non_root() -> None: + """(a) SET_PROPERTY applies and inverts exactly on both the root + container and a non-root one, restoring byte-identical program text.""" + space = TreeEditSpace() + base = parse_statements(PROGRAM) + assert base is not None + prop_idx = space.property_names.index("rest") + + def round_trip(stmt_idx: int) -> None: + rests = space.component_property_domains[base[stmt_idx].comp]["rest"] + old_idx = rests.index(base[stmt_idx].rest) + new_idx = next(i for i in range(len(rests)) if i != old_idx) + mutated = space.apply( + base, + Edit(ACTION_SET_PROPERTY, stmt_idx, prop_idx, target=new_idx), + INVENTORY, + ) + assert mutated is not None, f"stmt {stmt_idx} did not apply" + assert mutated[stmt_idx].rest == rests[new_idx] + assert mutated[stmt_idx].rest != base[stmt_idx].rest + validate(render_statements(mutated)) + restored = space.apply( + mutated, + Edit(ACTION_SET_PROPERTY, stmt_idx, prop_idx, target=old_idx), + INVENTORY, + ) + assert restored is not None + validate(render_statements(restored)) + # Byte-identical restoration -- the safe-inverse contract. + assert render_statements(restored) == PROGRAM + + # root (stmt 0): the guard that blocks REMOVE/REMOVE_CONTAINER on root + # must NOT block SET_PROPERTY -- this is exactly the reachability gap + # VAR1-01 proved (needs_direction_change on the root's own rest). + assert base[0].name == "root" + round_trip(0) + # non-root container ("inline_card" = Card([title]), stmt 1). + assert base[1].name == "inline_card" and base[1].has_list + round_trip(1) + + # Out-of-precondition edits fail closed: a leaf statement has no + # container property domain to index into. + leaf_idx = next(i for i, s in enumerate(base) if not s.has_list) + assert ( + space.apply( + base, Edit(ACTION_SET_PROPERTY, leaf_idx, prop_idx, target=0), INVENTORY + ) + is None + ) + # A same-value "mutation" is not a real change -- rejected, not silently + # applied as a no-op edit. + same_idx = space.component_property_domains["Stack"]["rest"].index(base[0].rest) + assert ( + space.apply( + base, + Edit(ACTION_SET_PROPERTY, 0, prop_idx, target=same_idx), + INVENTORY, + ) + is None + ) + + +def test_set_property_rejects_out_of_domain_value_via_real_parser() -> None: + """(b) An illegal property value is rejected by the real parser/validator + -- never silently applied -- even when a (mistaken) pack declares it as + part of the domain. This proves the rejection is not merely an index + bounds-check: the final full-program re-validation is the actual + fail-closed backstop.""" + import slm_training.dsl.pack as pack_mod + from slm_training.dsl.pack import get_pack + + base_pack = get_pack("openui") + bogus_pack = replace( + base_pack, + pack_id="tree-edit-bogus-rest", + component_property_domains={ + **base_pack.component_property_domains, + # ``, [`` is not valid grammar for the direction/rest argument + # (an unterminated list literal) -- a pack authoring mistake + # that must still fail closed through the real parser. + "Stack": {"rest": (', "column"', ", [")}, + }, + ) + pack_mod.register_pack(bogus_pack) + try: + space = TreeEditSpace(pack_id=bogus_pack.pack_id) + prop_idx = space.property_names.index("rest") + statements = parse_statements('root = Stack([], "column")') + assert statements is not None + bogus_idx = space.component_property_domains["Stack"]["rest"].index(", [") + result = space.apply( + statements, + Edit(ACTION_SET_PROPERTY, 0, prop_idx, target=bogus_idx), + [], + ) + assert result is None + finally: + pack_mod._PACKS.pop(bogus_pack.pack_id, None) + + +def test_checkpoint_format2_fails_closed_and_migration_preserves_logits( + tmp_path, +) -> None: + """(c) A genuine format-2 checkpoint (N_ACTIONS=11, pre-VAR1-02 width) + fails closed with a clear pointer to the migration path. (d) Migrating + it to format 3 produces bit-identical action logits on the 11 + pre-existing rows; only the new (SET_PROPERTY) row differs.""" + records = [ + ExampleRecord( + id="a", + prompt="Hero card with title, body, and a CTA button.", + openui=PROGRAM, + placeholders=INVENTORY, + ) + ] + cfg = TreeEditDiffusionConfig( + d_model=32, n_heads=4, context_layers=1, denoiser_layers=1, + dropout=0.0, seed=5, + ) + model = TreeEditDiffusionModel.from_records(records, config=cfg, device="cpu") + assert model.policy.action_head.out_features == 12 # current N_ACTIONS + path = tmp_path / "ckpt.pt" + model.save(path) + + # Simulate a genuine format-2 checkpoint: truncate action_head to the + # pre-VAR1-02 width (11 rows) and stamp format_version=2. + payload = torch.load(path, map_location="cpu", weights_only=False) + payload["format_version"] = 2 + sd = payload["state_dict"] + old_w = sd["policy.action_head.weight"][:11].clone() + old_b = sd["policy.action_head.bias"][:11].clone() + sd["policy.action_head.weight"] = old_w + sd["policy.action_head.bias"] = old_b + v2_path = tmp_path / "ckpt_v2.pt" + torch.save(payload, v2_path) + (tmp_path / "ckpt_v2.tokenizer.json").write_text( + (tmp_path / "ckpt.tokenizer.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + # (c) Fails closed, pointing at the migration path. + with pytest.raises(ValueError, match="format_version=2") as excinfo: + TreeEditDiffusionModel.from_checkpoint(v2_path, device="cpu") + assert "migrate_tree_edit_checkpoint" in str(excinfo.value) + + from slm_training.models.checkpoint_migrate import migrate_tree_edit_checkpoint + + out_path = tmp_path / "ckpt_migrated_v3.pt" + report = migrate_tree_edit_checkpoint( + source_checkpoint=v2_path, output_checkpoint=out_path + ) + assert report["source_format_version"] == 2 + assert report["output_format_version"] == 3 + assert report["preserved_action_head_rows"] == 11 + + migrated = TreeEditDiffusionModel.from_checkpoint(out_path, device="cpu") + new_w = migrated.policy.action_head.weight + new_b = migrated.policy.action_head.bias + assert new_w.shape[0] == 12 + assert torch.allclose(new_w[:11], old_w) + assert torch.allclose(new_b[:11], old_b) + + # (d) Bit-identical logits on the pre-existing rows: the migrated + # checkpoint's non-action_head weights are copied verbatim from the same + # trained model as the untouched original, so a forward pass over the + # same input must match exactly on rows 0..10; only the new row 11 (a + # fresh random init) is expected to differ. + reference = TreeEditDiffusionModel.from_checkpoint(path, device="cpu") + migrated.eval() + reference.eval() + prompt_text = migrated._format_context(records[0].prompt) + with torch.no_grad(): + ctx_m, ctx_pad_m = migrated._encode_context([prompt_text]) + out_m = migrated.policy( + migrated._state_batch([PROGRAM]), migrated.tokenizer.pad_id, ctx_m, ctx_pad_m + ) + ctx_r, ctx_pad_r = reference._encode_context([prompt_text]) + out_r = reference.policy( + reference._state_batch([PROGRAM]), reference.tokenizer.pad_id, ctx_r, ctx_pad_r + ) + assert torch.allclose(out_m["action"][:, :11], out_r["action"][:, :11]) + assert not torch.allclose(out_m["action"][:, 11], out_r["action"][:, 11])