From 8ee5485a609a000b6bb66fa07a30c34237b33c9d Mon Sep 17 00:00:00 2001 From: alice Date: Tue, 1 Sep 2026 13:53:31 +0000 Subject: [PATCH 1/2] scsd: re-emit op 0x02 to the coordinator at the ADD transition (vms-a84d) The booted-node cluster join stalls at barrier step 5/12: the second existing member (VAX1) never pushes its disk-DLM rebuild toward the joining OVMX node, so it withholds its own op-0b#5 and the coordinator never releases 0c#5. Root cause (directly observed on the inter-VAX channel already captured in the run-1025 pcap; three-way grounded against the vax3-2to3 reference): the coordinator emits its op-0x12 "rebuild toward the new member" relay to the other existing member ONLY in response to a FRESH joiner op-0x02 landed AT the transition. In the reference this is exact -- joiner op-0x02 -> COORD op-0x12 relay 0.3ms later -> that member rebuilds and clears its barrier step. OVMX's op-0x02 to the coordinator is one-shot on the JOIN_CFG2_DELAY_MS (4.9s) timer (cfg_sent/joiner_cfg2_sent), decoupled from the coordinator's transitions: it DID fire COORD's relay once, but for the pre-epoch transition; the OVMX-ADD epoch opens afterward and gets no fresh op-0x02, so COORD never relays OVMX into VAX1's rebuild set. Fix: when the coordinator opens an ADD transition (op-0x09 XITION) while we are the joiner, re-emit a fresh op-0x02 to the coordinator, once per ADD-epoch. The op-0x02 CONTENT is byte-identical to the deferred one (proven byte-equal to a real joiner's op-0x02); only its TIMING changes. No fabricated lock/held state -- INV-6 clean; the gate-clearing member rebuild is a pure cat-0x82 echo OVMX already performs for the coordinator. Joiner-path-guarded (rx->do_connect && !appeared_after_join), so the member/bystander and OVMX<->OVMX (H2) paths are untouched. H2 cannot exercise the real COORD->VAX1 relay, so CI-green is necessary-not- sufficient; the lab re-fire vs vaxlab-2 is the actual proof. All 26 vmsscs unit tests pass (incl. the 141k-frame scsd_wire replay). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01PhM3QcmFEY3p8YNnHGaYwP --- src/vmsscs/scsd.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/src/vmsscs/scsd.c b/src/vmsscs/scsd.c index 6d698683d..8f0aa22d2 100644 --- a/src/vmsscs/scsd.c +++ b/src/vmsscs/scsd.c @@ -1089,6 +1089,12 @@ struct peer_state { * peer-initiated connection and answer its config reactively * on the member VC, like OVMX_PURE_SERVER. */ int joiner_cfg2_sent; /* vms-760: the DEFERRED op 0x02 config/topology went out */ + int cfg2_reemit_pending; /* vms-a84d: a FRESH op 0x02 is armed for this (coordinator) + * peer, to be sent at the transition moment so the + * coordinator relays us (op 0x12) into the other members' + * rebuild set. Same content as the deferred op 0x02. */ + uint32_t cfg2_reemit_epoch; /* vms-a84d: the ADD-transition epoch we last armed a + * re-emit for (one re-emit per epoch, never per frame). */ int rejoin_credit_first_sent; /* vms-46f: OVMX's op-6 special-credit request rode the * coordinator's SCS$DIRECTORY connection AHEAD of op 0x02 * (spec 4(O.17), Davis pp. 2-43/2-44). Gates the deferred @@ -7537,6 +7543,41 @@ static void scsd_sysap_msg_input(struct scs_cdt *cdt, const void *msg, size_t ms "UNKNOWN-CLASS"); fflush(stdout); } + /* vms-a84d: ARM a FRESH op 0x02 to the coordinator at the ADD + * transition. The deferred op 0x02 (CMCONFIG2) is one-shot on a + * 4.9s timer, decoupled from the coordinator's transitions. A real + * 2->3 join shows the coordinator emits its op 0x12 relay to the + * OTHER existing member ONLY in response to a FRESH joiner op 0x02 + * landed AT the transition (vax3-2to3: joiner op 0x02 @+34.7634 -> + * COORD op 0x12 relay to the member @+34.7637, 0.3ms; the member + * then rebuilds toward the joiner and clears its own barrier step). + * In the 1025 stall OVMX's single op 0x02 DID fire the relay -- but + * for the pre-epoch transition; epoch-0xE (the OVMX-ADD) opened + * after it and got no fresh op 0x02, so COORD never relayed OVMX + * into VAX1's rebuild set, VAX1 withheld its 0b#5, and OVMX stalled + * at barrier step 5. Re-emit once per ADD-epoch, only while WE are + * the joiner. The CONTENT is byte-identical to the deferred op 0x02 + * (honest -- proven byte-equal to a real joiner's); only the TIMING + * changes. The send itself rides the poll-loop choke point below. */ + if (mv.opcode == SCS_MEMBER_OP_XITION && + ps->xition_class == SCS_MEMBER_CLASS_ADD && + rx->do_connect && !ps->appeared_after_join && + ps->barrier_epoch != 0) { + struct peer_state *coord = cm_pick_coordinator(rx->peers); + if (coord != NULL && coord->cfg_sent && + coord->cfg2_reemit_epoch != ps->barrier_epoch) { + coord->cfg2_reemit_epoch = ps->barrier_epoch; + coord->cfg2_reemit_pending = 1; + log_ts(stdout); + printf(" SCSD-I-CFG2REEMIT, ADD transition epoch=0x%08X" + " opened -- arming a FRESH op 0x02 to coordinator" + " node %u so it relays us into the other members'" + " rebuild set (vms-a84d)\n", + (unsigned)ps->barrier_epoch, + (unsigned)(peer_node_number(coord) & 0x03ff)); + fflush(stdout); + } + } /* vms-e81 (T1.1): the barrier must arm for EVERY transition, * not just our own join. * @@ -11921,6 +11962,51 @@ static void scsd_handle_frame(struct scsd_rx *rx, const uint8_t *buf, ssize_t n) } } + /* vms-a84d: the transition-phase RE-EMIT of op 0x02, armed by the + * ADD-XITION handler. Fires a FRESH op 0x02 to the coordinator so it emits + * the op 0x12 relay for THIS epoch (the deferred one-shot already went out + * for an earlier one). Same content + same VC as the deferred op 0x02 -- + * honest, re-timed. Independent of joiner_cfg2_sent so it survives the + * one-shot latch; guarded once-per-epoch by cfg2_reemit_epoch in the arm. */ + if (rx->do_connect && !ps->appeared_after_join && + ps->cfg_sent && ps->cfg2_reemit_pending && + (getenv("OVMX_CFG2_ALL") != NULL || + cm_pick_coordinator(rx->peers) == ps)) { + struct scs_member_params mp; + memset(&mp, 0, sizeof(mp)); + memcpy(mp.dst_mac, ps_port_addr(ps), 6); + memcpy(mp.src_mac, rx->our_hw_mac, 6); + memcpy(mp.src_logical, rx->our_src_logical, 6); + memcpy(mp.peer_logical, ps_sys_addr(ps), 6); + mp.remote_conid = ps->cfg_remote_conid; + mp.local_conid = ps->cfg_local_conid; + mp.incarnation = ps->incarnation; + mp.recv_ack = ps->vc.seq.recv_seq; + mp.send_seq = scs_seq_advance(&ps->vc.seq); + mp.sysap_send_msg = ps->sysap_send++; + mp.sysap_ack_msg = ps->sysap_recv; + cm_apply_rejoin_form(&mp); /* vms-2f3 */ + uint8_t rframe[SCS_MEMBER_FRAME_LEN]; + if (scs_member_build_config(&mp, rframe) == 0 && + send_frame_vc(rx->sock, rx->ifindex, ps, ps->pb, + "transition-phase RE-EMIT op 0x02 (config/topology)", + rframe, sizeof(rframe)) > 0) { + scs_vc_record_sent(&ps->vc, mp.send_seq, monotonic_ms()); + ps->cfg2_reemit_pending = 0; + rx->cm_config_frames++; + log_ts(stdout); + printf(" SCSD-I-CFG2REEMIT, sent FRESH op 0x02 to coordinator node %u" + " at the transition (epoch=0x%08X) on VC local=0x%08X" + " remote=0x%08X (send_msg=%u) -- expect its op 0x12 relay to" + " the other member(s) so they rebuild toward us (vms-a84d)\n", + (unsigned)(peer_node_number(ps) & 0x03ff), + (unsigned)ps->cfg2_reemit_epoch, + (unsigned)ps->cfg_local_conid, (unsigned)ps->cfg_remote_conid, + mp.sysap_send_msg); + fflush(stdout); + } + } + /* vms-760 PURE-SERVER disk-client: stop-and-wait retransmit of the * current outstanding PS drive frame if VAX1's response has not * arrived. Reuses the stored send_seq (never advances -- 760mscp From 4a6f5b35057f58350af9412950fe90513f15f866 Mon Sep 17 00:00:00 2001 From: alice Date: Tue, 1 Sep 2026 14:26:18 +0000 Subject: [PATCH 2/2] scsd: send the transition op-0x02 re-emit SYNCHRONOUSLY, before the barrier (vms-a84d) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lab re-fire #1029b tested the epoch re-emit in isolation (collision landed on the coordinator's VC, XITION opened, SCSD-I-CFG2REEMIT fired) and exposed a timing inversion: the re-emit was armed on the XITION but SENT from the poll-loop choke point, which fires on the JOIN_CFG2_DELAY_MS (4.9s) cadence -- so it landed at +35.26, ~4s AFTER the barrier had already run steps 1-5 and stalled at +31.2. The coordinator needs the fresh op-0x02 BEFORE it fans out the barrier; 4s late it is ignored, no op-0x12 relay, VAX1 never rebuilds, barrier stuck at 5/12. The reference (vax3-2to3) sends its relay-triggering op-0x02 0.04s BEFORE the barrier (+34.7634 op-0x02 -> +34.807 barrier). Match that: send the fresh op-0x02 SYNCHRONOUSLY in the XITION handler, the instant the ADD transition opens (~1s before the barrier fan-out), instead of deferring to the poll loop. Drops the poll-loop block and the cfg2_reemit_pending flag; keeps cfg2_reemit_epoch as the once-per-epoch guard. Content still byte-identical to the deferred op-0x02 (INV-6 clean); only the timing changes. Joiner-path-guarded. All 26 vmsscs unit tests pass. ⚠ Still a candidate: whether a correctly-timed fresh op-0x02 actually triggers the coordinator's op-0x12 relay is coordinator-internal and unproven -- the lab re-fire is the arbiter. #1029b could not test it because the send was 4s late. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01PhM3QcmFEY3p8YNnHGaYwP --- src/vmsscs/scsd.c | 116 +++++++++++++++++++++------------------------- 1 file changed, 54 insertions(+), 62 deletions(-) diff --git a/src/vmsscs/scsd.c b/src/vmsscs/scsd.c index 8f0aa22d2..1ead7d35d 100644 --- a/src/vmsscs/scsd.c +++ b/src/vmsscs/scsd.c @@ -1089,12 +1089,11 @@ struct peer_state { * peer-initiated connection and answer its config reactively * on the member VC, like OVMX_PURE_SERVER. */ int joiner_cfg2_sent; /* vms-760: the DEFERRED op 0x02 config/topology went out */ - int cfg2_reemit_pending; /* vms-a84d: a FRESH op 0x02 is armed for this (coordinator) - * peer, to be sent at the transition moment so the - * coordinator relays us (op 0x12) into the other members' - * rebuild set. Same content as the deferred op 0x02. */ - uint32_t cfg2_reemit_epoch; /* vms-a84d: the ADD-transition epoch we last armed a - * re-emit for (one re-emit per epoch, never per frame). */ + uint32_t cfg2_reemit_epoch; /* vms-a84d: the ADD-transition epoch we last SENT a fresh + * op 0x02 re-emit for (one per epoch, never per frame). The + * re-emit is sent SYNCHRONOUSLY in the XITION handler so it + * precedes the coordinator's barrier fan-out (a poll-loop + * deferral lands ~4s late, after the barrier commits). */ int rejoin_credit_first_sent; /* vms-46f: OVMX's op-6 special-credit request rode the * coordinator's SCS$DIRECTORY connection AHEAD of op 0x02 * (spec 4(O.17), Davis pp. 2-43/2-44). Gates the deferred @@ -7564,18 +7563,56 @@ static void scsd_sysap_msg_input(struct scs_cdt *cdt, const void *msg, size_t ms rx->do_connect && !ps->appeared_after_join && ps->barrier_epoch != 0) { struct peer_state *coord = cm_pick_coordinator(rx->peers); - if (coord != NULL && coord->cfg_sent && + if (coord != NULL && coord->cfg_sent && coord->pb != NULL && + coord->cfg_remote_conid != 0 && coord->cfg2_reemit_epoch != ps->barrier_epoch) { - coord->cfg2_reemit_epoch = ps->barrier_epoch; - coord->cfg2_reemit_pending = 1; - log_ts(stdout); - printf(" SCSD-I-CFG2REEMIT, ADD transition epoch=0x%08X" - " opened -- arming a FRESH op 0x02 to coordinator" - " node %u so it relays us into the other members'" - " rebuild set (vms-a84d)\n", - (unsigned)ps->barrier_epoch, - (unsigned)(peer_node_number(coord) & 0x03ff)); - fflush(stdout); + /* Send the fresh op 0x02 SYNCHRONOUSLY here, in the + * frame-receive path, the instant the ADD XITION opens -- + * so it precedes the coordinator's barrier fan-out. The + * reference joiner's relay-triggering op 0x02 is 0.04s + * BEFORE the barrier (vax3-2to3: op 0x02 +34.7634 -> + * barrier +34.807); a poll-loop-deferred send lands ~4s + * LATE, after the barrier commits at step 5, and the + * coordinator ignores it (#1029b: re-emit @+35.26, barrier + * stalled @+31.2). Content byte-identical to the deferred + * op 0x02 -- only the timing changes. Rides the SAME VC the + * coordinator's config burst used (cfg_*_conid). */ + struct scs_member_params mp; + memset(&mp, 0, sizeof(mp)); + memcpy(mp.dst_mac, ps_port_addr(coord), 6); + memcpy(mp.src_mac, rx->our_hw_mac, 6); + memcpy(mp.src_logical, rx->our_src_logical, 6); + memcpy(mp.peer_logical, ps_sys_addr(coord), 6); + mp.remote_conid = coord->cfg_remote_conid; + mp.local_conid = coord->cfg_local_conid; + mp.incarnation = coord->incarnation; + mp.recv_ack = coord->vc.seq.recv_seq; + mp.send_seq = scs_seq_advance(&coord->vc.seq); + mp.sysap_send_msg = coord->sysap_send++; + mp.sysap_ack_msg = coord->sysap_recv; + cm_apply_rejoin_form(&mp); /* vms-2f3 */ + uint8_t rframe[SCS_MEMBER_FRAME_LEN]; + if (scs_member_build_config(&mp, rframe) == 0 && + send_frame_vc(rx->sock, (int)rx->ifindex, coord, coord->pb, + "transition-phase RE-EMIT op 0x02 (config/topology)", + rframe, sizeof(rframe)) > 0) { + scs_vc_record_sent(&coord->vc, mp.send_seq, monotonic_ms()); + coord->cfg2_reemit_epoch = ps->barrier_epoch; + rx->cm_config_frames++; + log_ts(stdout); + printf(" SCSD-I-CFG2REEMIT, ADD transition epoch=0x%08X" + " opened -- sent a FRESH op 0x02 to coordinator node" + " %u SYNCHRONOUSLY (before the barrier fan-out) on VC" + " local=0x%08X remote=0x%08X (send_msg=%u) so it" + " relays us into the other members' rebuild set" + " (vms-a84d)\n", + (unsigned)ps->barrier_epoch, + (unsigned)(peer_node_number(coord) & 0x03ff), + (unsigned)coord->cfg_local_conid, + (unsigned)coord->cfg_remote_conid, + mp.sysap_send_msg); + fflush(stdout); + } } } /* vms-e81 (T1.1): the barrier must arm for EVERY transition, @@ -11962,51 +11999,6 @@ static void scsd_handle_frame(struct scsd_rx *rx, const uint8_t *buf, ssize_t n) } } - /* vms-a84d: the transition-phase RE-EMIT of op 0x02, armed by the - * ADD-XITION handler. Fires a FRESH op 0x02 to the coordinator so it emits - * the op 0x12 relay for THIS epoch (the deferred one-shot already went out - * for an earlier one). Same content + same VC as the deferred op 0x02 -- - * honest, re-timed. Independent of joiner_cfg2_sent so it survives the - * one-shot latch; guarded once-per-epoch by cfg2_reemit_epoch in the arm. */ - if (rx->do_connect && !ps->appeared_after_join && - ps->cfg_sent && ps->cfg2_reemit_pending && - (getenv("OVMX_CFG2_ALL") != NULL || - cm_pick_coordinator(rx->peers) == ps)) { - struct scs_member_params mp; - memset(&mp, 0, sizeof(mp)); - memcpy(mp.dst_mac, ps_port_addr(ps), 6); - memcpy(mp.src_mac, rx->our_hw_mac, 6); - memcpy(mp.src_logical, rx->our_src_logical, 6); - memcpy(mp.peer_logical, ps_sys_addr(ps), 6); - mp.remote_conid = ps->cfg_remote_conid; - mp.local_conid = ps->cfg_local_conid; - mp.incarnation = ps->incarnation; - mp.recv_ack = ps->vc.seq.recv_seq; - mp.send_seq = scs_seq_advance(&ps->vc.seq); - mp.sysap_send_msg = ps->sysap_send++; - mp.sysap_ack_msg = ps->sysap_recv; - cm_apply_rejoin_form(&mp); /* vms-2f3 */ - uint8_t rframe[SCS_MEMBER_FRAME_LEN]; - if (scs_member_build_config(&mp, rframe) == 0 && - send_frame_vc(rx->sock, rx->ifindex, ps, ps->pb, - "transition-phase RE-EMIT op 0x02 (config/topology)", - rframe, sizeof(rframe)) > 0) { - scs_vc_record_sent(&ps->vc, mp.send_seq, monotonic_ms()); - ps->cfg2_reemit_pending = 0; - rx->cm_config_frames++; - log_ts(stdout); - printf(" SCSD-I-CFG2REEMIT, sent FRESH op 0x02 to coordinator node %u" - " at the transition (epoch=0x%08X) on VC local=0x%08X" - " remote=0x%08X (send_msg=%u) -- expect its op 0x12 relay to" - " the other member(s) so they rebuild toward us (vms-a84d)\n", - (unsigned)(peer_node_number(ps) & 0x03ff), - (unsigned)ps->cfg2_reemit_epoch, - (unsigned)ps->cfg_local_conid, (unsigned)ps->cfg_remote_conid, - mp.sysap_send_msg); - fflush(stdout); - } - } - /* vms-760 PURE-SERVER disk-client: stop-and-wait retransmit of the * current outstanding PS drive frame if VAX1's response has not * arrived. Reuses the stored send_seq (never advances -- 760mscp