From 67138bd0516e99216581a0ae5a840094f64b1c66 Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Wed, 8 Jul 2026 16:39:42 -0500 Subject: [PATCH 1/8] [DOC] Doc for Pathname#relative_path_from --- pathname_builtin.rb | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/pathname_builtin.rb b/pathname_builtin.rb index 9b31b73b87af43..a03c829ef3ff8e 100644 --- a/pathname_builtin.rb +++ b/pathname_builtin.rb @@ -895,19 +895,41 @@ def each_child(with_directory=true, &b) children(with_directory).each(&b) end + # :markup: markdown # - # Returns a relative path from the given +base_directory+ to the receiver. + # call-seq: + # relative_path_from(source) -> new_pathname # - # If +self+ is absolute, then +base_directory+ must be absolute too. + # Returns a pathname containing the relative filesystem path from the given `source` + # to the path in `self`; + # `source` must be a directory path or a pathname containing a directory path: # - # If +self+ is relative, then +base_directory+ must be relative too. + # ```ruby + # Pathname('.').relative_path_from('doc/language') # => # + # Pathname('doc/language').relative_path_from('.') # => # + # Pathname('doc').relative_path_from('doc/language') # => # + # ``` # - # This method doesn't access the filesystem. It assumes no symlinks. + # The paths need not exist: + # + # ```ruby + # Pathname('nosuch').relative_path_from('nosuch/foo/bar/baz') + # # => # + # ``` # - # ArgumentError is raised when it cannot find a relative path. + # The two paths must be either both absolute or both relative: # - # Note that this method does not handle situations where the case sensitivity - # of the filesystem in use differs from the operating system default. + # ```ruby + # Pathname('/var').relative_path_from('/etc') # => # + # Pathname('/var').relative_path_from('doc') # Raises ArgumentError + # Pathname('doc').relative_path_from('/etc') # Raises ArgumentError + # ``` + # + # Raises an exception if there is no such relative path: + # + # ```ruby + # Pathname('foo').relative_path_from('..') # Raises ArgumentError + # ``` # def relative_path_from(base_directory) base_directory = Pathname.new(base_directory) unless base_directory.is_a? Pathname From 898751482d372b1a7861c6e42e1e426314d3850d Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Wed, 8 Jul 2026 16:03:49 -0500 Subject: [PATCH 2/8] [DOC] Doc for Pathname#relative? --- pathname_builtin.rb | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pathname_builtin.rb b/pathname_builtin.rb index a03c829ef3ff8e..ab3132725db541 100644 --- a/pathname_builtin.rb +++ b/pathname_builtin.rb @@ -581,17 +581,25 @@ def mountpoint? end end - # The opposite of Pathname#absolute? + # :markup: markdown + # + # call-seq: + # relative? -> true or false # - # It returns +false+ if the pathname begins with a slash. + # Returns whether `self` contains a relative path: # - # p = Pathname.new('/im/sure') - # p.relative? - # #=> false + # ```ruby + # Pathname('lib').relative? # => true + # Pathname('/home').relative? # => false + # ``` + # + # The result is OS-dependent for some paths: + # + # ```ruby + # Pathname('C:/').relative? # => false # On Windows. + # Pathname('C:/').relative? # => true # Elsewhere. + # ``` # - # p = Pathname.new('not/so/sure') - # p.relative? - # #=> true def relative? !absolute? end From 61b5f059cbccd3e74a14e0e2e4d6db3bdfdb4a47 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 9 Jul 2026 10:59:26 +0900 Subject: [PATCH 3/8] Remove dead RARRAY_EMBED_LEN_MAX --- include/ruby/internal/core/rarray.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/ruby/internal/core/rarray.h b/include/ruby/internal/core/rarray.h index b47b63d51b05e2..d70535e9d9ba41 100644 --- a/include/ruby/internal/core/rarray.h +++ b/include/ruby/internal/core/rarray.h @@ -45,7 +45,6 @@ /** @cond INTERNAL_MACRO */ #define RARRAY_EMBED_FLAG RARRAY_EMBED_FLAG #define RARRAY_EMBED_LEN_MASK RARRAY_EMBED_LEN_MASK -#define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX #define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT /** @endcond */ #define RARRAY_LEN rb_array_len /**< @alias{rb_array_len} */ From 9003f9f21a380827e6a6895245ecb2e84fcba5f3 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 9 Jul 2026 00:13:05 -0700 Subject: [PATCH 4/8] ZJIT: Avoid eager stack spills for inlined frame pushes (#17387) PushInlineFrame keeps caller stack slots lazy. When an inlined callee side-exits, install a caller JITFrame with a stack map on the exit path so rb_zjit_materialize_frames can reconstruct older frames without eagerly spilling at frame push. Co-authored-by: Alan Wu Co-authored-by: Kevin Menard --- vm.c | 11 +- zjit.h | 27 +- zjit/bindgen/src/main.rs | 1 + zjit/src/backend/arm64/mod.rs | 5 +- zjit/src/backend/lir.rs | 223 ++++++++++-- zjit/src/backend/x86_64/mod.rs | 5 +- zjit/src/codegen.rs | 469 +++++++++++++++----------- zjit/src/codegen_tests.rs | 35 ++ zjit/src/cruby_bindings.inc.rs | 1 + zjit/src/hir.rs | 28 +- zjit/src/hir/opt_tests.rs | 600 ++++++++++++++++----------------- zjit/src/hir/tests.rs | 20 +- 12 files changed, 862 insertions(+), 563 deletions(-) diff --git a/vm.c b/vm.c index 85a22d40048261..86c0e3391553a6 100644 --- a/vm.c +++ b/vm.c @@ -2858,17 +2858,22 @@ rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t // jit_return, so this restores older ZJIT frames from stack maps. int32_t stack_size = (int32_t)jit_frame->stack_size; if (stack_size > 0) { - VALUE *stack = cfp->sp - stack_size; + VALUE *stack = cfp->sp; for (int32_t i = 0; i < stack_size; i++) { VALUE entry = jit_frame->stack[i]; if (ZJIT_STACK_MAP_VREG_P(entry)) { // Decode a native stack slot offset generated by ZJIT's backend. // It's an offset from NATIVE_BASE_PTR, which is copied into // cfp->jit_return, to the encoded stack slot. - stack[i] = ((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_VREG_INDEX(entry)]; + stack--; + *stack = ((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_VREG_INDEX(entry)]; + } + else if (ZJIT_STACK_MAP_SKIP_P(entry)) { + stack -= ZJIT_STACK_MAP_SKIP_SIZE(entry); } else { - stack[i] = entry; + stack--; + *stack = entry; } } } diff --git a/zjit.h b/zjit.h index 34ebdc095c6be9..586cf28a20c598 100644 --- a/zjit.h +++ b/zjit.h @@ -9,10 +9,12 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif -// Stack map entries are either immediate Ruby VALUEs or tagged native-stack -// locations. Stack maps never contain heap VALUEs, so 0x08 is available: it is -// not Qfalse (0), and its low 3 bits are zero, so RB_SPECIAL_CONST_P is false. +// Stack map entries are either immediate Ruby VALUEs, tagged native-stack +// locations, or tagged skip counts. Stack maps never contain heap VALUEs, so +// these tags are available: they are not Qfalse (0), and their low 3 bits are +// zero, so RB_SPECIAL_CONST_P is false. #define ZJIT_STACK_MAP_VREG_TAG 0x08 +#define ZJIT_STACK_MAP_SKIP_TAG 0x10 #define ZJIT_STACK_MAP_TAG_MASK 0xff #define ZJIT_STACK_MAP_SHIFT 8 @@ -28,6 +30,18 @@ ZJIT_STACK_MAP_VREG_INDEX(VALUE entry) return entry >> ZJIT_STACK_MAP_SHIFT; } +static inline bool +ZJIT_STACK_MAP_SKIP_P(VALUE entry) +{ + return (entry & ZJIT_STACK_MAP_TAG_MASK) == ZJIT_STACK_MAP_SKIP_TAG; +} + +static inline size_t +ZJIT_STACK_MAP_SKIP_SIZE(VALUE entry) +{ + return entry >> ZJIT_STACK_MAP_SHIFT; +} + // JITFrame is defined here as the single source of truth and imported into // Rust via bindgen. C code reads fields directly; Rust uses an impl block. typedef struct zjit_jit_frame { @@ -43,12 +57,11 @@ typedef struct zjit_jit_frame { // Always false for C frames. bool materialize_block_code; - // Number of Ruby stack slots described by stack[]. - // rb_zjit_materialize_frames() copies them to cfp->sp - stack_size. + // Number of stack map entries in stack[]. uint32_t stack_size; // Flexible array of stack map entries. Each entry is either an immediate - // VALUE or a tagged native-stack index from cfp->jit_return for a value - // kept by the JIT. + // VALUE, a tagged native-stack index from cfp->jit_return for a value + // kept by the JIT, or a tagged count of VM stack slots to skip. VALUE stack[]; } zjit_jit_frame_t; diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 801663a81f1c8e..95f7ee87cc5e9c 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -334,6 +334,7 @@ fn main() { .allowlist_type("zjit_struct_offsets") .allowlist_var("ZJIT_STACK_MAP_SHIFT") .allowlist_var("ZJIT_STACK_MAP_VREG_TAG") + .allowlist_var("ZJIT_STACK_MAP_SKIP_TAG") .allowlist_var("ZJIT_JIT_RETURN_C_FRAME") .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_complex_p") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 5c5c5cd3fa87f4..a91d0e8b375896 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1644,6 +1644,7 @@ impl Assembler { let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); asm.stack_state.num_spill_slots = num_stack_slots; + asm.stack_state.num_side_exit_stack_map_slots = asm.side_exit_stack_map_slots(&assignments); let stack_slot_count = asm.stack_state.stack_slot_count(); if stack_slot_count > Self::MAX_FRAME_STACK_SLOTS { return Err(CompileError::NativeStackTooLarge); @@ -1670,7 +1671,7 @@ impl Assembler { } // Update FrameSetup slot_count now that StackState knows the - // register allocator spill count. + // register allocator spill and side-exit capture counts. trace_compile_phase("count_stack_slots", || { for block in asm.basic_blocks.iter_mut() { for insn in block.insns.iter_mut() { @@ -1781,7 +1782,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } })); + let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], stack_map: None, recompile: None } })); asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index e5962b84dca775..f47fa06bdd5038 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -5,7 +5,7 @@ use std::mem::take; use std::rc::Rc; use crate::bitset::BitSet; use crate::codegen::{perf_symbol_range_start, perf_symbol_range_end, register_with_perf}; -use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, VALUE, ZJIT_STACK_MAP_SHIFT, ZJIT_STACK_MAP_VREG_TAG, vm_stack_canary, YarvInsnIdx, zjit_jit_frame, local_size_and_idx_to_ep_offset}; +use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, VALUE, ZJIT_STACK_MAP_SHIFT, ZJIT_STACK_MAP_SKIP_TAG, ZJIT_STACK_MAP_VREG_TAG, vm_stack_canary, YarvInsnIdx, zjit_jit_frame, local_size_and_idx_to_ep_offset}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -586,6 +586,10 @@ pub struct SideExit { pub stack: Vec, pub locals: Vec, pub iseq: IseqPtr, + /// Stack map for older inlined frames that are not written directly by this + /// side exit. The current frame's stack and locals are still handled by + /// `stack` and `locals` above. + pub stack_map: Option, /// If set, the side exit will profile the current instruction and invalidate /// the compiled ISEQ for recompilation. pub recompile: Option, @@ -922,11 +926,18 @@ pub enum Insn { } macro_rules! target_for_each_operand_impl { - ($self:expr, $visit_many:ident) => { + ($self:expr, $visit_one:ident, $visit_many:ident, $reborrow:ident) => { match $self { Target::SideExit(data) => { visit_many!(data.exit.stack); visit_many!(data.exit.locals); + if let Some(StackMap { stack, .. }) = $reborrow!(data.exit.stack_map) { + for entry in stack { + if let StackMapEntry::Opnd(opnd) = entry { + visit_one!(opnd); + } + } + } } Target::Block(edge) => { visit_many!(edge.args); @@ -956,17 +967,17 @@ macro_rules! for_each_operand_impl { Insn::Jz(target) | Insn::Label(target) | Insn::LeaJumpTarget { target, .. } => { - target_for_each_operand_impl!(target, $visit_many); + target_for_each_operand_impl!(target, $visit_one, $visit_many, $reborrow); } // `target` is behind a Box. `$reborrow` turns the box field into a `&`/`&mut Target` // matching the iterator, so the same operand-walk works for both. Insn::PatchPoint(data) => { - target_for_each_operand_impl!($reborrow!(data.target), $visit_many); + target_for_each_operand_impl!($reborrow!(data.target), $visit_one, $visit_many, $reborrow); } Insn::Joz(opnd, target) | Insn::Jonz(opnd, target) => { visit_one!(opnd); - target_for_each_operand_impl!(target, $visit_many); + target_for_each_operand_impl!(target, $visit_one, $visit_many, $reborrow); } Insn::BakeString(_) | @@ -1021,7 +1032,11 @@ macro_rules! for_each_operand_impl { // Option` matching the iterator, so the same operand-walk works for // both. if let Some(StackMap { stack, .. }) = $reborrow!(data.stack_map) { - visit_many!(stack); + for entry in stack { + if let StackMapEntry::Opnd(opnd) = entry { + visit_one!(opnd); + } + } } } // only iterate over preserved in the const iterator @@ -1484,15 +1499,18 @@ const JIT_FRAME_OFFSET_FROM_JIT_RETURN: usize = 1; /// | +-------------------------+ | /// v | opnds.first() | v /// +-------------------------+ -/// ^ | register spill slot 0 | ^ +/// ^ | allocator spill slot 0 | ^ /// | +-------------------------+ | /// | | ... | | stack_idx for "slot N" in StackState::stack_map_index_for_spill +/// num_spill_slots | +-------------------------+ | +/// v | allocator spill slot X | | +/// +-------------------------+ | +/// ^ | side-exit stack-map | | +/// | | capture slot 0 | | /// | +-------------------------+ | -/// num_spill_slots | | register spill slot N | v -/// | +-------------------------+ -/// | | ... | -/// | +-------------------------+ -/// v | register spill slot X | +/// num_side_exit_stack_map_slots | | ... | | +/// | +-------------------------+ | +/// v | capture slot X | v /// +-------------------------+ /// | FrameSetup align slot | if needed /// +-------------------------+ @@ -1505,17 +1523,21 @@ pub struct StackState { /// The number of stack slots needed by register allocator spills. pub(crate) num_spill_slots: usize, + + /// The maximum number of stack slots needed to capture side-exit stack-map + /// operands that cannot be encoded directly. + pub(crate) num_side_exit_stack_map_slots: usize, } impl StackState { /// Initialize an empty stack state. fn new() -> Self { - StackState { stack_base_idx: 0, num_spill_slots: 0 } + StackState { stack_base_idx: 0, num_spill_slots: 0, num_side_exit_stack_map_slots: 0 } } /// Initialize a stack state with a fixed number of reserved stack slots. fn new_with_stack_slots(stack_base_idx: usize) -> Self { - StackState { stack_base_idx, num_spill_slots: 0 } + StackState { stack_base_idx, num_spill_slots: 0, num_side_exit_stack_map_slots: 0 } } /// Reserve native stack slots for JITFrame storage and stack-allocated operands. @@ -1527,9 +1549,9 @@ impl StackState { } /// Return the total number of native stack slots used for the frame's - /// reserved data and register allocator spills. + /// reserved data, register allocator spills, and side-exit captures. pub(crate) fn stack_slot_count(&self) -> usize { - self.stack_base_idx + self.num_spill_slots + self.stack_base_idx + self.num_spill_slots + self.num_side_exit_stack_map_slots } /// Return the stack-map index for a VReg stored below StackState-managed @@ -1551,7 +1573,13 @@ impl StackState { /// Return a stack index for a register saved by handle_caller_saved_regs(). fn stack_idx_for_caller_saved_reg(&self, caller_saved_reg_idx: usize) -> usize { let frame_alignment_slots = self.stack_slot_count() % 2; - self.num_spill_slots + frame_alignment_slots + caller_saved_reg_idx + self.num_spill_slots + self.num_side_exit_stack_map_slots + frame_alignment_slots + caller_saved_reg_idx + } + + /// Return a stack index reserved for side-exit stack-map register capture. + fn stack_idx_for_side_exit_stack_map(&self, slot_idx: usize) -> usize { + assert!(slot_idx < self.num_side_exit_stack_map_slots); + self.num_spill_slots + slot_idx } /// Convert a stack index to the `disp` of the stack slot @@ -1572,12 +1600,10 @@ impl StackState { } /// Stack map to materialize Ruby stack slots from JIT-kept values. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct StackMap { /// Ruby stack slots to reconstruct if this frame is materialized. - /// Each operand must be either an immediate Ruby VALUE or a VReg whose - /// final register/spill location will be encoded after register allocation. - stack: Vec, + stack: Vec, /// Heap-allocated JITFrame whose trailing stack map storage receives the /// encoded entries once this CCall's register allocation is known. jit_frame: *const zjit_jit_frame, @@ -1586,6 +1612,21 @@ pub struct StackMap { frame_depth: usize, } +impl StackMap { + pub fn new(stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) -> Self { + Self { stack, jit_frame, frame_depth } + } +} + +/// Entry in a JITFrame stack map. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum StackMapEntry { + /// Immediate Ruby VALUE or VReg to materialize. + Opnd(Opnd), + /// Number of VM stack slots to skip when materializing across inlined frames. + Skip(usize), +} + /// Initial capacity for asm.insns vector const ASSEMBLER_INSNS_CAPACITY: usize = 256; @@ -2392,8 +2433,8 @@ impl Assembler // Build a set of VRegIds that can be referenced by JITFrame for materializing the VM stack let stack_vreg_ids: HashSet = if let Some(StackMap { stack, .. }) = &stack_map { - stack.iter().filter_map(|opnd| match opnd { - Opnd::VReg { idx, .. } => Some(*idx), + stack.iter().filter_map(|entry| match entry { + StackMapEntry::Opnd(Opnd::VReg { idx, .. }) => Some(*idx), _ => None, }).collect() } else { @@ -2437,18 +2478,28 @@ impl Assembler if let Some(StackMap { stack, jit_frame, frame_depth }) = stack_map { assert_eq!(unsafe { (*jit_frame).stack_size } as usize, stack.len()); - for (idx, stack_opnd) in stack.iter().enumerate() { - let entry = match stack_opnd { - Opnd::UImm(value) => { - let value = VALUE(*value as usize); + for (idx, stack_entry) in stack.iter().enumerate() { + let entry = match *stack_entry { + StackMapEntry::Opnd(Opnd::Value(value)) => { + // TODO: Investigate using a constant pool to track any value reference in the stack map + assert!(value.special_const_p(), "StackMap should only materialize immediate VALUEs, but got: {value:?}"); + value + } + StackMapEntry::Opnd(Opnd::UImm(value)) => { + let value = VALUE(value as usize); // TODO: Investigate using a constant pool to track any value reference in the stack map assert!(value.special_const_p(), "StackMap should only materialize immediate VALUEs, but got: {value:?}"); value } - Opnd::VReg { idx: vreg, .. } => { - let vreg_stack_index = match assignments[*vreg].expect("StackMap VReg should have an allocation") { + StackMapEntry::Skip(size) => { + let encoded = (size << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_SKIP_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap skip should not look like an immediate VALUE"); + VALUE(encoded) + } + StackMapEntry::Opnd(Opnd::VReg { idx: vreg, .. }) => { + let vreg_stack_index = match assignments[vreg].expect("StackMap VReg should have an allocation") { Allocation::Reg(_) | Allocation::Fixed(_) => { - let caller_saved_reg_idx = survivors.iter().position(|&survivor_id| survivor_id == *vreg).unwrap(); + let caller_saved_reg_idx = survivors.iter().position(|&survivor_id| survivor_id == vreg).unwrap(); let stack_idx = self.stack_state.stack_idx_for_caller_saved_reg(caller_saved_reg_idx); self.stack_state.stack_map_index_for_spill(stack_idx, frame_depth) } @@ -2462,7 +2513,7 @@ impl Assembler debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap VReg should not look like an immediate VALUE"); VALUE(encoded) } - _ => unreachable!("unexpected operand in StackMap: {stack_opnd:?}"), + _ => unreachable!("unexpected entry in StackMap: {stack_entry:?}"), }; unsafe { (*jit_frame.cast_mut()).stack.as_mut_ptr().add(idx).write(entry); } } @@ -2567,6 +2618,40 @@ impl Assembler } } + /// Return the maximum number of stack-map entries that any side exit needs + /// to copy into reserved native stack slots. + pub fn side_exit_stack_map_slots(&self, assignments: &[Option]) -> usize { + self.block_order().into_iter().fold(0, |max_slots, block_id| { + let block = &self.basic_blocks[block_id.0]; + block.insns.iter().fold(max_slots, |max_slots, insn| { + let slots = insn.target().map(|target| Self::side_exit_target_stack_map_slots(target, assignments)).unwrap_or(0); + max_slots.max(slots) + }) + }) + } + + fn side_exit_target_stack_map_slots(target: &Target, assignments: &[Option]) -> usize { + let Target::SideExit(data) = target else { + return 0; + }; + let Some(StackMap { stack, .. }) = &data.exit.stack_map else { + return 0; + }; + + stack.iter().filter(|entry| match entry { + StackMapEntry::Opnd(Opnd::Value(value)) => !value.special_const_p(), + StackMapEntry::Opnd(Opnd::UImm(value)) => !VALUE(*value as usize).special_const_p(), + StackMapEntry::Opnd(Opnd::VReg { idx, .. }) => { + matches!( + assignments[idx.to_usize()].expect("StackMap VReg should have an allocation"), + Allocation::Reg(_) | Allocation::Fixed(_) + ) + } + StackMapEntry::Opnd(Opnd::Reg(_)) => true, + _ => false, + }).count() + } + /// Walk every instruction and replace VReg operands with the physical /// register (or stack slot) from the allocation assignments. fn rewrite_instructions(&mut self, assignments: &[Option]) { @@ -2679,9 +2764,75 @@ impl Assembler /// Returns the exit code as a list of instructions to be appended after the main /// code is linearized and split. pub fn compile_exits(&mut self) -> Vec { + fn immediate_stack_map_value(opnd: Opnd) -> Option { + let value = match opnd { + Opnd::Value(value) => value, + Opnd::UImm(value) => VALUE(value as usize), + _ => unreachable!("unexpected immediate StackMap operand: {opnd:?}"), + }; + value.special_const_p().then_some(value) + } + + fn encode_stack_map_index(asm: &Assembler, stack_idx: usize, frame_depth: usize) -> VALUE { + let vreg_stack_index = asm.stack_state.stack_map_index_for_spill(stack_idx, frame_depth); + let encoded = (vreg_stack_index << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_VREG_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap VReg should not look like an immediate VALUE"); + VALUE(encoded) + } + + fn capture_stack_map_opnd(asm: &mut Assembler, opnd: Opnd, capture_idx: &mut usize, frame_depth: usize) -> VALUE { + let stack_idx = asm.stack_state.stack_idx_for_side_exit_stack_map(*capture_idx); + *capture_idx += 1; + let capture_slot = Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx: stack_idx.try_into().unwrap(), num_bits: 64 }, + disp: 0, + num_bits: 64, + }); + let opnd = if matches!(opnd, Opnd::Reg(_)) { opnd.with_num_bits(64) } else { opnd }; + asm.store(capture_slot, opnd); + encode_stack_map_index(asm, stack_idx, frame_depth) + } + + fn compile_exit_stack_map(asm: &mut Assembler, stack_map: &StackMap) { + let StackMap { stack, jit_frame, frame_depth } = stack_map; + let jit_frame = *jit_frame; + assert_eq!(unsafe { (*jit_frame).stack_size } as usize, stack.len()); + + let mut capture_idx = 0; + for (idx, stack_entry) in stack.iter().enumerate() { + let entry = match *stack_entry { + StackMapEntry::Opnd(Opnd::Value(_) | Opnd::UImm(_)) => { + let StackMapEntry::Opnd(opnd) = *stack_entry else { unreachable!() }; + immediate_stack_map_value(opnd) + .unwrap_or_else(|| capture_stack_map_opnd(asm, opnd, &mut capture_idx, *frame_depth)) + } + StackMapEntry::Skip(size) => { + let encoded = (size << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_SKIP_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap skip should not look like an immediate VALUE"); + VALUE(encoded) + } + StackMapEntry::Opnd(Opnd::Mem(Mem { base: MemBase::Stack { stack_idx, .. }, disp, .. })) => { + assert_eq!(disp, 0, "StackMap stack slot should not have a displacement"); + encode_stack_map_index(asm, stack_idx.to_usize(), *frame_depth) + } + StackMapEntry::Opnd(Opnd::Reg(_)) => { + let StackMapEntry::Opnd(opnd) = *stack_entry else { unreachable!() }; + capture_stack_map_opnd(asm, opnd, &mut capture_idx, *frame_depth) + } + _ => unreachable!("unexpected entry in SideExit StackMap: {stack_entry:?}"), + }; + unsafe { (*jit_frame.cast_mut()).stack.as_mut_ptr().add(idx).write(entry); } + } + + assert!(capture_idx <= asm.stack_state.num_side_exit_stack_map_slots); + asm_comment!(asm, "install side-exit JITFrame for caller depth {}", frame_depth); + let jit_frame_slot = Opnd::mem(64, NATIVE_BASE_PTR, -((*frame_depth as i32 + 1) * SIZEOF_VALUE_I32)); + asm.store(jit_frame_slot, Opnd::const_ptr(jit_frame)); + } + /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { - let SideExit { pc, stack, locals, iseq, .. } = exit; + let SideExit { pc, stack, locals, iseq, stack_map, .. } = exit; // Side exit blocks are not part of the CFG at the moment, // so we need to manually ensure that patchpoints get padded @@ -2712,6 +2863,10 @@ impl Assembler asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); } } + + if let Some(stack_map) = stack_map { + compile_exit_stack_map(asm, stack_map); + } } /// Tear down the JIT frame and return to the interpreter. @@ -3846,9 +4001,9 @@ impl Assembler { /// HIR function currently reserves multiple native JITFrame slots, one per /// inlining depth, stack-map indexes must be encoded relative to the target /// frame's own cfp->jit_return. - pub fn stack_map(&mut self, stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) { + pub fn stack_map(&mut self, stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) { assert!(self.stack_map.is_none()); - self.stack_map = Some(StackMap { stack, jit_frame, frame_depth }); + self.stack_map = Some(StackMap::new(stack, jit_frame, frame_depth)); } pub fn store(&mut self, dest: Opnd, src: Opnd) { diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 963a177dd8125a..e83ad9f2e7ad73 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1155,6 +1155,7 @@ impl Assembler { let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); asm.stack_state.num_spill_slots = num_stack_slots; + asm.stack_state.num_side_exit_stack_map_slots = asm.side_exit_stack_map_slots(&assignments); let stack_slot_count = asm.stack_state.stack_slot_count(); if stack_slot_count > Self::MAX_FRAME_STACK_SLOTS { return Err(CompileError::NativeStackTooLarge); @@ -1181,7 +1182,7 @@ impl Assembler { } // Update FrameSetup slot_count now that StackState knows the - // register allocator spill count. + // register allocator spill and side-exit capture counts. trace_compile_phase("count_stack_slots", || { for block in asm.basic_blocks.iter_mut() { for insn in block.insns.iter_mut() { @@ -1391,7 +1392,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } })); + let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], stack_map: None, recompile: None } })); asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 6a610e2c4235e4..0ad066c5e2bc24 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -21,7 +21,7 @@ use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, SideExitTarget, Target, asm_ccall, asm_comment}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, SideExitTarget, StackMap, StackMapEntry, Target, asm_ccall, asm_comment}; use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; use crate::hir::{BlockHandler, CCallVariadicData, CCallWithFrameData, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendDirectData, SendFallbackReason}; use crate::hir_type::{types, Type}; @@ -528,7 +528,7 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func Insn::InvokeBuiltin { .. } => SideExitReason::UnhandledHIRInvokeBuiltin, _ => SideExitReason::UnhandledHIRUnknown(insn_id), }; - gen_side_exit(&mut jit, &mut asm, &reason, None, &function.frame_state(last_snapshot)); + gen_side_exit(&mut jit, &mut asm, function, &reason, None, &function.frame_state(last_snapshot)); // Don't bother generating code after a side-exit. We won't run it. // TODO(max): Generate ud2 or equivalent. break; @@ -615,8 +615,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), Insn::NewArray { elements, state } => gen_new_array(jit, asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, function, opnds!(elements), &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, function, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), Insn::AdjustBounds { index, length } => gen_adjust_bounds(asm, opnd!(index), opnd!(length)), @@ -626,49 +626,50 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), - Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, function, opnd!(val), &function.frame_state(*state)), &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), - Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, function, opnds!(strings), &function.frame_state(*state)), &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), - Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), - Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, function, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, function, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::StringEqual { left, right } => gen_string_equal(asm, opnd!(left), opnd!(right)), Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), - Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, function, *opt, opnds!(values), &function.frame_state(*state)), Insn::Param => unreachable!("block.insns should not have Insn::Param"), Insn::LoadArg { .. } => return Ok(()), // compiled in the LoadArg pre-pass above Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment - &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), - &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, cd, std::ptr::null(), &function.frame_state(state), reason), - &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, function, cd, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, function, cd, std::ptr::null(), &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), Insn::SendDirect(insn) => { let SendDirectData { cme, iseq, recv, args, kw_bits, block, state, .. } = &**insn; gen_send_iseq_direct( - cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), + cb, jit, asm, + function, *cme, *iseq, opnd!(recv), opnds!(args), *kw_bits, &function.frame_state(*state), *block, ) } Insn::PushInlineFrame { cme, iseq, recv, args, blockiseq, state, .. } => { - no_output!(gen_push_inline_frame(jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), *blockiseq)) + no_output!(gen_push_inline_frame(jit, asm, function, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), *blockiseq)) }, Insn::PopInlineFrame { iseq, argc, state } => { no_output!(gen_pop_inline_frame(asm, *iseq, *argc, &function.frame_state(*state))) }, - &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), - Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), - Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), - Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), unsafe { &**bf }, *leaf, opnds!(args)), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, function, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, function, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, function, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, function, &function.frame_state(*state), unsafe { &**bf }, *leaf, opnds!(args)), &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), - Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), Insn::FloatAdd { recv, other, state } => gen_float_add(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::FloatSub { recv, other, state } => gen_float_sub(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::FloatMul { recv, other, state } => gen_float_mul(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), @@ -689,7 +690,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio // We only create FixnumLShift when we know the shift amount statically and it's in [0, // 63]. let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; - gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + gen_fixnum_lshift(jit, asm, function, opnd!(left), shift_amount, &function.frame_state(state)) } &Insn::FixnumRShift { left, right } => { // We only create FixnumRShift when we know the shift amount statically and it's in [0, @@ -697,13 +698,13 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; gen_fixnum_rshift(asm, opnd!(left), shift_amount) } - &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state)), &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), &Insn::IsMethodCfunc { val, cd, cfunc, state } => gen_is_method_cfunc(asm, opnd!(val), cd, cfunc, &function.frame_state(state)), &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), - &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, function, opnd!(val), &function.frame_state(state)), &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), Insn::Test { val } => gen_test(asm, opnd!(val)), Insn::RefineType { val, .. } => opnd!(val), @@ -713,55 +714,55 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } &Insn::GuardType { val, guard_type, state, recompile } => { let val_type = function.type_of(val); - gen_guard_type(jit, asm, opnd!(val), val_type, guard_type, recompile, &function.frame_state(state)) + gen_guard_type(jit, asm, function, opnd!(val), val_type, guard_type, recompile, &function.frame_state(state)) } - &Insn::GuardBitEquals { val, expected, ref reason, state, recompile } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, **reason, recompile, &function.frame_state(state)), - &Insn::GuardAnyBitSet { val, mask, ref reason, state, recompile, .. } => gen_guard_any_bit_set(jit, asm, opnd!(val), mask, **reason, recompile, &function.frame_state(state)), - &Insn::GuardNoBitsSet { val, mask, ref reason, state, .. } => gen_guard_no_bits_set(jit, asm, opnd!(val), mask, **reason, &function.frame_state(state)), - &Insn::GuardLess { left, right, ref reason, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), **reason, &function.frame_state(state)), - &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), - Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + &Insn::GuardBitEquals { val, expected, ref reason, state, recompile } => gen_guard_bit_equals(jit, asm, function, opnd!(val), expected, **reason, recompile, &function.frame_state(state)), + &Insn::GuardAnyBitSet { val, mask, ref reason, state, recompile, .. } => gen_guard_any_bit_set(jit, asm, function, opnd!(val), mask, **reason, recompile, &function.frame_state(state)), + &Insn::GuardNoBitsSet { val, mask, ref reason, state, .. } => gen_guard_no_bits_set(jit, asm, function, opnd!(val), mask, **reason, &function.frame_state(state)), + &Insn::GuardLess { left, right, ref reason, state } => gen_guard_less(jit, asm, function, opnd!(left), opnd!(right), **reason, &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state)), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, function, invariant, &function.frame_state(*state))), Insn::CCall { cfunc, recv, args, name, owner: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), Insn::CCallWithFrame(insn) => { let CCallWithFrameData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; - gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + gen_ccall_with_frame(jit, asm, function, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) } Insn::CCallVariadic(insn) => { let CCallVariadicData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; - gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + gen_ccall_variadic(jit, asm, function, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) } Insn::GetIvar { self_val, id, ic, state } => gen_getivar(asm, opnd!(self_val), *id, *ic, &function.frame_state(*state)), - Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), - Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, function, *id, opnd!(val), &function.frame_state(*state))), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, function, *id, &function.frame_state(*state)), &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), - &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, function, ep_offset, level, &function.frame_state(state)), &Insn::SetLocal { val, ep_offset, level, .. } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), - Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), - Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), - Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), - Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), - Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, function, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, function, *ic, &function.frame_state(*state)), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, function, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, function, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, function, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), - Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, reason, *recompile, &function.frame_state(*state))), - Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, *value_type, &function.frame_state(*state)), + Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, function, reason, *recompile, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, function, *value_type, &function.frame_state(*state)), Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), - Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), - Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, function, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), + Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, function, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), Insn::GetSpecialSymbol { symbol_type, state } => gen_getspecial_symbol(asm, *symbol_type, &function.frame_state(*state)), Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), - &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), + &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, function, &function.frame_state(state))), Insn::BreakPoint => no_output!(asm.breakpoint()), Insn::Unreachable => no_output!(asm.abort()), &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, - &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, - &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, function, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, function, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, - &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, - &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, function, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, function, opnd!(val), &function.frame_state(state)) }, &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, - &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state))) }, Insn::LoadPC => gen_load_pc(asm), Insn::LoadEC => gen_load_ec(), Insn::LoadSP => gen_load_sp(), @@ -771,13 +772,13 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(jit, asm, opnd!(recv), opnd!(val), function.type_of(val))), &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), - Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), - Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), - &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), - Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, function, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, function, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, function, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, function, opnds!(elements), &function.frame_state(*state)), &Insn::IsA { val, class } => gen_is_a(jit, asm, opnd!(val), opnd!(class)), - &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, opnds!(elements), &function.frame_state(state)), - &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, function, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, function, opnds!(elements), &function.frame_state(state)), &Insn::Throw { state, .. } => return Err(state), &Insn::CondBranch { .. } | &Insn::Jump { .. } | Insn::Entries { .. } => unreachable!(), @@ -810,7 +811,7 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { ep_opnd } -fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { +fn gen_defined(jit: &JITState, asm: &mut Assembler, function: &Function, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { match op_type as defined_type { DEFINED_YIELD => { // `lep_level` was precomputed at HIR construction so we can materialize the local EP @@ -823,7 +824,7 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, } _ => { // Save the PC and SP because the callee may allocate or call #respond_to? - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // TODO: Inline the cases for each op_type // Call vm_defined(ec, reg_cfp, op_type, obj, v) @@ -877,13 +878,13 @@ fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { /// Get the block parameter as a Proc, write it to the environment, /// and mark the flag as modified. -fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, function: &Function, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { gen_prepare_leaf_call_with_gc(asm, state); // Bail out if write barrier is required. let ep = gen_get_ep(asm, level); let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); - asm.jnz(jit, side_exit(jit, state, SideExitReason::BlockParamWbRequired)); + asm.jnz(jit, side_exit(jit, function, state, SideExitReason::BlockParamWbRequired)); // Convert block handler to Proc. let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); @@ -903,36 +904,36 @@ fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, le asm.load(Opnd::mem(VALUE_BITS, ep, offset)) } -fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { +fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { asm.cmp(left, right); - asm.jge(jit, side_exit(jit, state, reason)); + asm.jge(jit, side_exit(jit, function, state, reason)); left } -fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { +fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { asm.cmp(left, right); - asm.jl(jit, side_exit(jit, state, SideExitReason::GuardGreaterEq)); + asm.jl(jit, side_exit(jit, function, state, SideExitReason::GuardGreaterEq)); left } -fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, function: &Function, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; } // Anything could be called on const_missing - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) } -fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { +fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, function: &Function, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_get_ev_const(ec: EcPtr, klass: VALUE, id: ID, allow_nil: VALUE) -> VALUE; } // Constant lookup can raise and run arbitrary Ruby code via const_missing. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_get_ev_const, EC, klass, id.0.into(), allow_nil) } @@ -943,7 +944,7 @@ fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { asm.csel_z(Qtrue.into(), Qfalse.into()) } -fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec) -> lir::Opnd { +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec) -> lir::Opnd { // +2 for ec, self assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", @@ -953,7 +954,7 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf gen_prepare_leaf_call_with_gc(asm, state); } else { // Anything can happen inside builtin functions - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); } let mut cargs = vec![EC]; @@ -964,9 +965,9 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf } /// Record a patch point that should be invalidated on a given invariant -fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, function: &Function, invariant: &Invariant, state: &FrameState) { let invariant = *invariant; - let exit = build_side_exit(jit, state); + let exit = build_side_exit(jit, function, state); // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. asm.patch_point(Target::SideExit(Box::new(SideExitTarget { exit, reason: PatchPoint(invariant) })), invariant, jit.version); @@ -1018,6 +1019,7 @@ pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invari fn gen_ccall_with_frame( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cfunc: *const u8, name: ID, recv: Opnd, @@ -1027,7 +1029,7 @@ fn gen_ccall_with_frame( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); - gen_stack_overflow_check(jit, asm, state, state.stack_size()); + gen_stack_overflow_check(jit, asm, function, state, state.stack_size()); let args_with_recv_len = args.len() + 1; if args_with_recv_len > C_ARG_OPNDS.len() { @@ -1039,7 +1041,7 @@ fn gen_ccall_with_frame( // to account for the receiver and arguments (and block arguments if any) gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, caller_stack_size); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); gen_spill_locals(jit, asm, state); let block_handler_specval = if let Some(BlockHandler::BlockIseq(block_iseq)) = block { @@ -1112,6 +1114,7 @@ fn gen_block_handler_specval(asm: &mut Assembler, blockiseq: IseqPtr) -> lir::Op fn gen_ccall_variadic( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cfunc: *const u8, name: ID, recv: Opnd, @@ -1121,7 +1124,7 @@ fn gen_ccall_variadic( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); - gen_stack_overflow_check(jit, asm, state, state.stack_size()); + gen_stack_overflow_check(jit, asm, function, state, state.stack_size()); let args_with_recv_len = args.len() + 1; @@ -1133,7 +1136,7 @@ fn gen_ccall_variadic( // to account for the receiver and arguments (like gen_ccall_with_frame does) gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, caller_stack_size); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); gen_spill_locals(jit, asm, state); let block_handler_specval = if let Some(BlockHandler::BlockIseq(blockiseq)) = block { @@ -1188,9 +1191,9 @@ fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_i } /// Emit an uncached instance variable store -fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, function: &Function, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); if ic.is_null() { asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); } else { @@ -1199,20 +1202,20 @@ fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: } } -fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) } -fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); } /// Look up global variables -fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, state: &FrameState) -> Opnd { // `Warning` module's method `warn` can be called when reading certain global variables - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_gvar_get, id.0.into()) } @@ -1225,25 +1228,25 @@ fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { } /// Set global variables -fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, val: Opnd, state: &FrameState) { // When trace_var is used, setting a global variable can cause exceptions - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_gvar_set, id.0.into(), val); } /// Side-exit into the interpreter -fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, recompile: Option, state: &FrameState) { - asm.jmp(side_exit_with_recompile(jit, state, *reason, recompile)); +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, function: &Function, reason: &SideExitReason, recompile: Option, state: &FrameState) { + asm.jmp(side_exit_with_recompile(jit, function, state, *reason, recompile)); } /// Emit a special object lookup -fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, value_type: SpecialObjectType, state: &FrameState) -> Opnd { +fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, function: &Function, value_type: SpecialObjectType, state: &FrameState) -> Opnd { // rb_vm_get_special_object for CBASE/CONST_BASE can call rb_singleton_class, // which allocates (may trigger GC) and can raise TypeError on non-class // receivers (e.g. `123.instance_eval { Const = 1 }`). Treat as non-leaf so // the PC is saved for GC and stack/locals are spilled for rescue. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // Get the EP of the current CFP and load it into a register let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); @@ -1287,7 +1290,7 @@ fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> O asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) } -fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // Check for interrupts // see RUBY_VM_CHECK_INTS(ec) macro asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); @@ -1295,7 +1298,7 @@ fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameSt // signal_exec, or rb_postponed_job_flush. let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG)); asm.test(interrupt_flag, interrupt_flag); - asm.jnz(jit, side_exit(jit, state, SideExitReason::Interrupt)); + asm.jnz(jit, side_exit(jit, function, state, SideExitReason::Interrupt)); } fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { @@ -1303,13 +1306,13 @@ fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd asm_ccall!(asm, rb_hash_resurrect, val) } -fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, function: &Function, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_hash_aref, hash, key) } -fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, function: &Function, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_hash_aset, hash, key, val); } @@ -1318,13 +1321,13 @@ fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameStat asm_ccall!(asm, rb_ary_push, array, val); } -fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) } -fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) } @@ -1332,11 +1335,11 @@ fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) } -fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { +fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, function: &Function, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { // rb_vm_check_match is not leaf unless flag is VM_CHECKMATCH_TYPE_WHEN. // See also: leafness_of_checkmatch() and check_match() if flag != VM_CHECKMATCH_TYPE_WHEN { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); } unsafe extern "C" { @@ -1346,8 +1349,8 @@ fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Op asm_ccall!(asm, rb_vm_check_match, EC, target, pattern, flag.into()) } -fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_ary_concat, left, right); } @@ -1465,6 +1468,7 @@ fn gen_param(asm: &mut Assembler, _idx: usize) -> lir::Opnd { fn gen_send( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1472,7 +1476,7 @@ fn gen_send( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -1488,6 +1492,7 @@ fn gen_send( fn gen_send_forward( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1495,7 +1500,7 @@ fn gen_send_forward( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { @@ -1512,13 +1517,14 @@ fn gen_send_forward( fn gen_send_without_block( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, state: &FrameState, reason: SendFallbackReason, ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; @@ -1536,6 +1542,7 @@ fn gen_send_without_block( fn gen_push_inline_frame( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cme: *const rb_callable_method_entry_t, iseq: IseqPtr, recv: Opnd, @@ -1545,7 +1552,7 @@ fn gen_push_inline_frame( ) { let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); - gen_stack_overflow_check(jit, asm, state, stack_growth); + gen_stack_overflow_check(jit, asm, function, state, stack_growth); // Save cfp->pc and cfp->sp for the caller frame. // Cannot use gen_prepare_non_leaf_call because we need special SP math. @@ -1554,7 +1561,6 @@ fn gen_push_inline_frame( gen_save_sp(asm, stack_size); gen_spill_locals(jit, asm, state); - gen_spill_stack(jit, asm, state); // This mirrors vm_caller_setup_arg_block() for the `blockiseq != NULL` case. // The HIR specialization guards ensure we will only reach here for literal blocks, @@ -1670,6 +1676,7 @@ fn gen_send_iseq_direct( cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, + function: &Function, cme: *const rb_callable_method_entry_t, iseq: IseqPtr, recv: Opnd, @@ -1682,16 +1689,17 @@ fn gen_send_iseq_direct( let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); - gen_stack_overflow_check(jit, asm, state, stack_growth); + gen_stack_overflow_check(jit, asm, function, state, stack_growth); // Save cfp->pc and cfp->sp for the caller frame // Can't use gen_prepare_non_leaf_call because we need special SP math. let stack_size = state.stack().len() - args.len() - 1; // -1 for receiver - let jit_frame = gen_write_jit_frame(asm, state, stack_size); + let stack_map = build_stack_map(jit, function, &state.with_stack_size(stack_size)); + let jit_frame = gen_write_jit_frame(asm, state, stack_map.len()); gen_save_sp(asm, stack_size); gen_spill_locals(jit, asm, state); - gen_stack_map(jit, asm, state, stack_size, jit_frame); + asm.stack_map(stack_map, jit_frame, state.depth); // This mirrors vm_caller_setup_arg_block() in for the `blockiseq != NULL` case. // The HIR specialization guards ensure we will only reach here for literal blocks, @@ -1820,13 +1828,14 @@ fn gen_send_iseq_direct( fn gen_invokeblock( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, state: &FrameState, reason: SendFallbackReason, ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call invokeblock"); unsafe extern "C" { @@ -1844,6 +1853,7 @@ fn gen_invokeblock( fn gen_invokeblock_ifunc( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, block_handler: Opnd, args: Vec, @@ -1851,7 +1861,7 @@ fn gen_invokeblock_ifunc( ) -> lir::Opnd { let _ = cd; // cd is not needed for the direct call - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); // Push args to memory so we can pass argv pointer let argv_ptr = gen_push_opnds(jit, asm, &args); @@ -1876,12 +1886,13 @@ fn gen_invokeblock_ifunc( fn gen_invokeproc( jit: &mut JITState, asm: &mut Assembler, + function: &Function, recv: Opnd, args: Vec, kw_splat: bool, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call invokeproc"); @@ -1903,6 +1914,7 @@ fn gen_invokeproc( fn gen_invokesuper( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1910,7 +1922,7 @@ fn gen_invokesuper( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call super with dynamic dispatch"); unsafe extern "C" { fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -1926,6 +1938,7 @@ fn gen_invokesuper( fn gen_invokesuperforward( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1933,7 +1946,7 @@ fn gen_invokesuperforward( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call super with dynamic dispatch (forwarding)"); unsafe extern "C" { fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -2058,11 +2071,12 @@ fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { fn gen_opt_newarray_hash( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { // `Array#hash` will hash the elements of the array. - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2085,10 +2099,11 @@ fn gen_opt_newarray_hash( fn gen_array_max( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); @@ -2111,10 +2126,11 @@ fn gen_array_max( fn gen_array_min( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); @@ -2136,11 +2152,12 @@ fn gen_array_min( fn gen_array_include( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, target: Opnd, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2163,12 +2180,13 @@ fn gen_array_include( fn gen_array_pack_buffer( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, fmt: Opnd, buffer: Option, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2195,11 +2213,12 @@ fn gen_array_pack_buffer( fn gen_dup_array_include( jit: &JITState, asm: &mut Assembler, + function: &Function, ary: VALUE, target: Opnd, state: &FrameState, ) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); unsafe extern "C" { fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; @@ -2264,6 +2283,7 @@ fn gen_is_a(jit: &mut JITState, asm: &mut Assembler, obj: Opnd, class: Opnd) -> fn gen_new_hash( jit: &mut JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { @@ -2283,7 +2303,7 @@ fn gen_new_hash( asm.store(Opnd::mem(VALUE_BITS, hash, RUBY_OFFSET_RHASH_IFNONE), Qnil.into()); hash } else { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); let argv = gen_push_opnds(jit, asm, &elements); asm_ccall!(asm, rb_hash_new_with_bulk_insert, elements.len().into(), argv) @@ -2294,13 +2314,14 @@ fn gen_new_hash( fn gen_new_range( jit: &JITState, asm: &mut Assembler, + function: &Function, low: lir::Opnd, high: lir::Opnd, flag: RangeType, state: &FrameState, ) -> lir::Opnd { // Sometimes calls `low.<=>(high)` - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // Call rb_range_new(low, high, flag) asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) @@ -2317,9 +2338,9 @@ fn gen_new_range_fixnum( asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) } -fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, state: &FrameState) -> lir::Opnd { // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_obj_alloc, val) } @@ -2391,25 +2412,25 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) { } /// Compile Fixnum + Fixnum -fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(jit, side_exit(jit, state, FixnumAddOverflow)); + asm.jo(jit, side_exit(jit, function, state, FixnumAddOverflow)); out_val } /// Compile Fixnum - Fixnum -fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(jit, side_exit(jit, state, FixnumSubOverflow)); + asm.jo(jit, side_exit(jit, function, state, FixnumSubOverflow)); asm.add(val_untag, Opnd::Imm(1)) } /// Compile Fixnum * Fixnum -fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Do some bitwise gymnastics to handle tag bits // x * y is translated to (x >> 1) * (y - 1) + 1 let left_untag = asm.rshift(left, Opnd::UImm(1)); @@ -2417,17 +2438,17 @@ fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, rig let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(jit, side_exit(jit, state, FixnumMultOverflow)); + asm.jo_mul(jit, side_exit(jit, function, state, FixnumMultOverflow)); asm.add(out_val, Opnd::UImm(1)) } /// Compile Fixnum / Fixnum -fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { gen_prepare_leaf_call_with_gc(asm, state); // Side exit if rhs is 0 asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); - asm.je(jit, side_exit(jit, state, FixnumDivByZero)); + asm.je(jit, side_exit(jit, function, state, FixnumDivByZero)); asm_ccall!(asm, rb_jit_fix_div_fix, left, right) } @@ -2520,14 +2541,14 @@ fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir } /// Compile Fixnum << Fixnum -fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { // Shift amount is known statically to be in the range [0, 63] assert!(shift_amount < 64); let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit let out_val = asm.lshift(in_val, shift_amount.into()); let unshifted = asm.rshift(out_val, shift_amount.into()); asm.cmp(in_val, unshifted); - asm.jne(jit, side_exit(jit, state, FixnumLShiftOverflow)); + asm.jne(jit, side_exit(jit, function, state, FixnumLShiftOverflow)); // Re-tag the output value let out_val = asm.add(out_val, 1.into()); out_val @@ -2542,10 +2563,10 @@ fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> asm.or(result, 1.into()) } -fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Check for left % 0, which raises ZeroDivisionError asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); - asm.je(jit, side_exit(jit, state, FixnumModByZero)); + asm.je(jit, side_exit(jit, function, state, FixnumModByZero)); asm_ccall!(asm, rb_fix_mod_fix, left, right) } @@ -2575,11 +2596,11 @@ fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) } -fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, state: &FrameState) -> lir::Opnd { // Load the value, then test for overflow and tag it let val = asm.load_mem(val); let shifted = asm.lshift(val, Opnd::UImm(1)); - asm.jo(jit, side_exit(jit, state, BoxFixnumOverflow)); + asm.jo(jit, side_exit(jit, function, state, BoxFixnumOverflow)); asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) } @@ -2713,33 +2734,33 @@ fn gen_has_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_typ } /// Compile a type check with a side exit -fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_type: Type, guard_type: Type, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, val_type: Type, guard_type: Type, recompile: Option, state: &FrameState) -> lir::Opnd { let is_known_heap_basic_object = val_type.is_subtype(types::HeapBasicObject); gen_incr_counter(asm, Counter::guard_type_count); if guard_type.is_subtype(types::Fixnum) { asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jz(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::Flonum) { // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::StaticSymbol) { // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG // Use 8-bit comparison like YJIT does. // If `val` is a constant (rare but possible), put it in a register to allow masking. let val = asm.load_imm(val); asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::NilClass) { asm.cmp(val, Qnil.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::TrueClass) { asm.cmp(val, Qtrue.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::FalseClass) { asm.cmp(val, Qfalse.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_immediate() { // All immediate types' guard should have been handled above panic!("unexpected immediate guard type: {guard_type}"); @@ -2750,7 +2771,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) let val = asm.load_mem(val); - let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side_exit = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); if !is_known_heap_basic_object { // Check if it's a special constant asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); @@ -2767,7 +2788,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t asm.cmp(klass, Opnd::Value(expected_class)); asm.jne(jit, side_exit); } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() { - let side = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); if !is_known_heap_basic_object { // Check special constant @@ -2786,7 +2807,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t asm.cmp(tag, Opnd::UImm(builtin_type as u64)); asm.jne(jit, side); } else if guard_type.bit_equal(types::HeapBasicObject) { - let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side_exit = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); asm.cmp(val, Opnd::Value(Qfalse)); asm.je(jit, side_exit.clone()); asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); @@ -2798,7 +2819,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { if matches!(reason, SideExitReason::GuardShape(_) ) { gen_incr_counter(asm, Counter::guard_shape_count); } @@ -2809,7 +2830,7 @@ fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), }; asm.cmp(val, expected_opnd); - asm.jnz(jit, side_exit_with_recompile(jit, state, reason, recompile)); + asm.jnz(jit, side_exit_with_recompile(jit, function, state, reason, recompile)); val } @@ -2824,18 +2845,18 @@ fn mask_to_opnd(mask: crate::hir::Const) -> Option { } /// Compile a bitmask check with a side exit if none of the masked bits are not set -fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_any_bit_set: unexpected hir::Const {mask:?}")); asm.test(val, mask_opnd); - asm.jz(jit, side_exit_with_recompile(jit, state, reason, recompile)); + asm.jz(jit, side_exit_with_recompile(jit, function, state, reason, recompile)); val } /// Compile a bitmask check with a side exit if any of the masked bits are set -fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { +fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_no_bits_set: unexpected hir::Const {mask:?}")); asm.test(val, mask_opnd); - asm.jnz(jit, side_exit(jit, state, reason)); + asm.jnz(jit, side_exit(jit, function, state, reason)); val } @@ -2930,16 +2951,22 @@ fn cfp_jit_return_for_depth(asm: &mut Assembler, depth: InlineDepth) -> Opnd { } } +fn jit_frame_next_pc(state: &FrameState) -> *const VALUE { + let opcode: usize = state.get_opcode().try_into().unwrap(); + unsafe { state.pc.offset(insn_len(opcode) as isize) } +} + +fn jit_frame_for_state(state: &FrameState, stack_map_size: usize) -> *const zjit_jit_frame { + JITFrame::new_iseq(jit_frame_next_pc(state), state.iseq, stack_map_size) +} + /// Save only the PC to CFP. Use this when you need to call gen_save_sp() /// immediately after with a custom stack size (e.g., gen_ccall_with_frame /// adjusts SP to exclude receiver and arguments). fn gen_write_jit_frame(asm: &mut Assembler, state: &FrameState, stack_map_size: usize) -> *const zjit_jit_frame { - let opcode: usize = state.get_opcode().try_into().unwrap(); - let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; - gen_incr_counter(asm, Counter::vm_write_jit_frame_count); asm_comment!(asm, "save JITFrame to CFP"); - let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, stack_map_size); + let jit_frame = jit_frame_for_state(state, stack_map_size); asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, jit_frame_slot_offset(state.depth)), Opnd::const_ptr(jit_frame)); // CFP_PC for a live JIT frame routes through the JITFrame on the native @@ -3010,13 +3037,23 @@ fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { } /// Spill the virtual stack onto the stack. -fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // This function does not call gen_save_sp() at the moment because // gen_send_without_block_direct() spills stack slots above SP for arguments. gen_incr_counter(asm, Counter::vm_write_stack_count); asm_comment!(asm, "spill stack"); - for (idx, &insn_id) in state.stack().enumerate() { - asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + + let mut offset = state.stack_size() as i32; + for entry in build_stack_map(jit, function, state) { + match entry { + StackMapEntry::Opnd(opnd) => { + offset -= 1; + asm.mov(Opnd::mem(64, SP, offset * SIZEOF_VALUE_I32), opnd); + } + StackMapEntry::Skip(skip) => { + offset -= skip as i32; + } + } } } @@ -3025,38 +3062,57 @@ fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { /// Direct JIT-to-JIT calls keep cfp->sp lazy, so this must publish SP before /// writing stack slots. Otherwise spilling the stack can overwrite frame /// metadata below the real VM-stack base. -fn gen_prepare_fallback_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_prepare_fallback_call(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, state.stack_size()); gen_spill_locals(jit, asm, state); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); } -/// Record the Ruby stack values needed to materialize this frame after the next -/// non-leaf C call. The actual JITFrame entries are encoded by the register -/// allocator, where VReg locations on the native stack are known. -fn gen_stack_map(jit: &JITState, asm: &mut Assembler, state: &FrameState, stack_size: usize, jit_frame: *const zjit_jit_frame) { +/// Build entries for Ruby stack values that need materialization. The actual +/// JITFrame entries are encoded by the register allocator, where VReg locations +/// on the native stack are known. +fn build_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> Vec { let mut stack = Vec::new(); - for &insn_id in state.stack().take(stack_size) { - let opnd = jit.get_opnd(insn_id); - // JITFrame only supports materializing Opnd::Value or Opnd::VReg out of the frame - assert!(matches!(opnd, Opnd::Value(_) | Opnd::VReg { .. }), "FrameState should only reference Opnd::Value or Opnd::VReg, but got: {opnd:?}"); - stack.push(opnd); + let mut current_state = state.clone(); + loop { + stack.extend(current_state.stack().rev().copied().map(|insn_id| { + let opnd = jit.get_opnd(insn_id); + assert!( + matches!(opnd, Opnd::Value(_) | Opnd::VReg { .. }), + "FrameState should only reference Opnd::Value or Opnd::VReg, but got: {opnd:?}", + ); + StackMapEntry::Opnd(opnd) + })); + + let Some(caller) = current_state.caller() else { + break; + }; + stack.push(StackMapEntry::Skip(inline_frame_stack_gap(current_state.iseq))); + current_state = function.frame_state(caller); } - asm.stack_map(stack, jit_frame, state.depth); + stack +} + +fn inline_frame_stack_gap(iseq: IseqPtr) -> usize { + // The extra slot is for the callee's receiver below its local table. + // We currently never map out the stack for `invokeblock`, which doesn't + // put a receiver on cfp->sp stack. + 1 + unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + VM_ENV_DATA_SIZE.to_usize() } /// Prepare for calling a C function that may call an arbitrary method. /// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. -fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // TODO: Lazily materialize caller frames when needed // Save PC for backtraces and allocation tracing // and SP to avoid marking uninitialized stack slots - let jit_frame = gen_prepare_call_with_gc(asm, state, false, state.stack_size()); + let stack_map = build_stack_map(jit, function, state); + let jit_frame = gen_prepare_call_with_gc(asm, state, false, stack_map.len()); - // Spill the virtual stack in case it raises an exception + // Remember the stack map in case it raises an exception // and the interpreter uses the stack for handling the exception - gen_stack_map(jit, asm, state, state.stack_size(), jit_frame); + asm.stack_map(stack_map, jit_frame, state.depth); // Spill locals in case the method looks at caller Bindings gen_spill_locals(jit, asm, state); @@ -3136,7 +3192,7 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C } /// Stack overflow check: fails if CFP<=SP at any point in the callee. -fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, function: &Function, state: &FrameState, stack_growth: usize) { asm_comment!(asm, "stack overflow check"); // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 // adds to the margin another control frame with `&bounds[1]`. @@ -3145,7 +3201,7 @@ fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &Fra let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); asm.cmp(CFP, stack_limit); - asm.jbe(jit, side_exit(jit, state, StackOverflow)); + asm.jbe(jit, side_exit(jit, function, state, StackOverflow)); } /// Convert ISEQ into High-level IR @@ -3179,14 +3235,14 @@ fn compile_iseq(iseq: IseqPtr) -> Result { } /// Build a Target::SideExit -fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { - let exit = build_side_exit(jit, state); +fn side_exit(jit: &JITState, function: &Function, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, function, state); Target::SideExit(Box::new(SideExitTarget { exit, reason })) } /// Build a Target::SideExit that optionally triggers exit_recompile on the exit path. -fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExitReason, recompile: Option) -> Target { - let mut exit = build_side_exit(jit, state); +fn side_exit_with_recompile(jit: &JITState, function: &Function, state: &FrameState, reason: SideExitReason, recompile: Option) -> Target { + let mut exit = build_side_exit(jit, function, state); exit.recompile = recompile.map(|_| SideExitRecompile { compiled_iseq: Opnd::Value(VALUE::from(jit.iseq())), insn_idx: state.insn_idx() as u32, @@ -3195,7 +3251,7 @@ fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExit } /// Build a side-exit context -fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { +fn build_side_exit(jit: &JITState, function: &Function, state: &FrameState) -> SideExit { let mut stack = Vec::new(); for &insn_id in state.stack() { stack.push(jit.get_opnd(insn_id)); @@ -3211,10 +3267,23 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { stack, locals, iseq: state.iseq, + stack_map: build_caller_stack_map(jit, function, state), recompile: None, } } +fn build_caller_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> Option { + let caller = state.caller()?; + let caller_state = function.frame_state(caller); + let stack_map = build_stack_map(jit, function, &caller_state); + if stack_map.is_empty() { + return None; + } + + let jit_frame = jit_frame_for_state(&caller_state, stack_map.len()); + Some(StackMap::new(stack_map, jit_frame, caller_state.depth)) +} + #[cfg(target_arch = "x86_64")] macro_rules! c_callable { ($(#[$outer:meta])* @@ -3632,15 +3701,15 @@ fn gen_push_opnds(jit: &JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::O argv } -fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, function: &Function, opt: usize, values: Vec, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); let first_opnd_ptr = gen_push_opnds(jit, asm, &values); asm_ccall!(asm, rb_reg_new_from_values, values.len().into(), first_opnd_ptr, opt.into()) } -fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, function: &Function, strings: Vec, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); let first_string_ptr = gen_push_opnds(jit, asm, &strings); asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr) @@ -3681,13 +3750,13 @@ fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, val asm_ccall!(asm, rb_str_setbyte, string, index, value) } -fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, function: &Function, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_str_buf_append, string, val) } -fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, function: &Function, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) } diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index efd158bfe3eeb4..51055badf29cad 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -6275,6 +6275,23 @@ fn test_inlined_method_with_invokeblock() { }); } +#[test] +fn test_inlined_method_with_invokeblock_raise_materializes_stack() { + with_inlining(|| { + assert_snapshot!(assert_inlines_allowing_exits(" + def callee = [1, 2, yield] + def test + callee { raise } + rescue + :rescued + end + + test + test + "), @":rescued"); + }); +} + #[test] fn test_inlined_method_with_block_param() { with_inlining(|| { @@ -6293,6 +6310,24 @@ fn test_inlined_method_with_block_param() { }); } +#[test] +fn test_inlined_method_that_forwards_block_arg_raise_materializes_stack() { + with_inlining(|| { + assert_snapshot!(assert_inlines_allowing_exits(" + def inner = yield + def callee(&block) = [1, 2, inner(&block)] + def test + callee { raise } + rescue + :rescued + end + + test + test + "), @":rescued"); + }); +} + #[test] fn test_inlined_method_that_forwards_block_arg() { // The callee captures a literal block in `&block` and forwards it on to diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index a43038e665ce2b..3a76b2a16f10e7 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -235,6 +235,7 @@ pub const VM_ENV_DATA_INDEX_FLAGS: u32 = 0; pub const VM_BLOCK_HANDLER_NONE: u32 = 0; pub const SHAPE_ID_NUM_BITS: u32 = 32; pub const ZJIT_STACK_MAP_VREG_TAG: u32 = 8; +pub const ZJIT_STACK_MAP_SKIP_TAG: u32 = 16; pub const ZJIT_STACK_MAP_SHIFT: u32 = 8; pub const ZJIT_JIT_RETURN_C_FRAME: u32 = 1; pub const RB_GC_ZJIT_FASTPATH_DATA_WORDS: u32 = 19; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 4d3aea54baca7c..028196d7d4ff59 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -4862,9 +4862,12 @@ impl Function { // callee sits one level deeper (caller_depth + 1). let caller_depth = self.frame_depth(state); + // The callee's perspective of the stack is with the receiver and arguments popped off. + let caller_stack_size = call_state.stack_size() - args.len() - 1; // -1 for receiver + let post_send_caller = self.new_insn(Insn::Snapshot { state: Box::new(call_state.with_stack_size(caller_stack_size)) }); let mode = AddIseqMode::Inlined { return_block: continuation, - caller: state, + caller: post_send_caller, depth: caller_depth + 1, jit_entry_idx: passed_opt_num, }; @@ -4992,6 +4995,10 @@ impl Function { // The original SendDirect result is now the continuation's return value param. self.make_equal_to(send_insn_id, return_val_param); + // Keep the caller FrameState that inlined callee Snapshots point at + // as a distinct Snapshot instead of rewriting the consumed SendDirect. + self.push_insn_id(block, post_send_caller); + // Insert PushLightweightFrame and jump to callee body entry. self.push_insn(block, Insn::PushInlineFrame { iseq, cme, recv, args: args.clone(), blockiseq, state, @@ -6847,7 +6854,7 @@ pub struct FrameState { stack: Vec, locals: Vec, - /// `InsnId` of the caller's call-site `Snapshot` for inlined frames; `None` + /// `InsnId` of the caller's post-send `Snapshot` for inlined frames; `None` /// for non-inlined frames. Stored as an instruction reference rather than /// an owned `FrameState` so that value remapping in the caller's `Snapshot` /// propagates here automatically, and so the caller's state has a single @@ -6882,6 +6889,13 @@ impl FrameState { state } + /// Return itself with a truncated stack. + pub fn with_stack_size(&self, stack_size: usize) -> Self { + let mut state = self.clone(); + state.stack.truncate(stack_size); + state + } + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. /// `original_argc` is the number of args originally on the stack (before processing). fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { @@ -6918,7 +6932,7 @@ impl FrameState { } /// Construct a `FrameState` for an inlined callee. `caller` is the `InsnId` - /// of the caller's call-site `Snapshot`; `depth` is this frame's inlining depth. + /// of the caller's post-send `Snapshot`; `depth` is this frame's inlining depth. fn inlined(iseq: IseqPtr, caller: InsnId, depth: InlineDepth) -> FrameState { FrameState { caller: Some(caller), depth, ..FrameState::new(iseq) } } @@ -6933,6 +6947,10 @@ impl FrameState { self.stack.iter() } + pub fn caller(&self) -> Option { + self.caller + } + /// Iterate over all local variables pub fn locals(&self) -> Iter<'_, InsnId> { self.locals.iter() @@ -7244,7 +7262,7 @@ enum AddIseqMode { Standalone, Inlined { return_block: BlockId, - /// The caller's call-site `Snapshot`. Allows side-exits to restore the outer frame. + /// The caller's post-send `Snapshot`. Allows side-exits to restore the outer frame. caller: InsnId, /// Inlining depth of every frame emitted for the callee. depth: InlineDepth, @@ -7315,7 +7333,7 @@ fn add_iseq_to_hir( let mut profiles = ProfileOracle::new(); // Build the initial FrameState for a block being translated. In inlined - // mode it carries the caller's call-site Snapshot and this frame's depth; + // mode it carries the caller's post-send Snapshot and this frame's depth; // because every Snapshot emitted for the callee is cloned from one of these // initial states, those values propagate to the whole inlined body without a // separate rewrite pass. diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index b82417d44aa7e7..595efc4e66b2a8 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -1546,11 +1546,11 @@ mod hir_opt_tests { v11:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v32:NilClass = Const Value(nil) + v33:NilClass = Const Value(nil) PushInlineFrame v20 (0x1038), v11 CheckInterrupts PopInlineFrame - Return v32 + Return v33 "); } @@ -1708,10 +1708,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:BasicObject = InvokeBuiltin rb_f_integer1, v20, v11 + v28:BasicObject = InvokeBuiltin rb_f_integer1, v20, v11 CheckInterrupts PopInlineFrame - Return v27 + Return v28 "); } @@ -1740,10 +1740,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 - v30:ArrayExact = NewArray + v31:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v30 + Return v31 "); } @@ -1768,13 +1768,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v18 (0x1038) - v25:Fixnum[1] = Const Value(1) - v33:Fixnum[2] = Const Value(2) + v26:Fixnum[1] = Const Value(1) + v34:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v60:Fixnum[3] = Const Value(3) + v61:Fixnum[3] = Const Value(3) CheckInterrupts PopInlineFrame - Return v60 + Return v61 "); } @@ -1800,12 +1800,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:Fixnum[2] = Const Value(2) + v28:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v53:Fixnum[5] = Const Value(5) + v54:Fixnum[5] = Const Value(5) CheckInterrupts PopInlineFrame - Return v53 + Return v54 "); } @@ -1833,10 +1833,10 @@ mod hir_opt_tests { v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v46:Fixnum[7] = Const Value(7) + v47:Fixnum[7] = Const Value(7) CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -1862,19 +1862,19 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) v44:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v44 (0x1038) - v56:Fixnum[1] = Const Value(1) - v66:Fixnum[2] = Const Value(2) - v76:Fixnum[3] = Const Value(3) - v86:Fixnum[4] = Const Value(4) - v101:ArrayExact = NewArray v56, v66, v76, v86 + v57:Fixnum[1] = Const Value(1) + v67:Fixnum[2] = Const Value(2) + v77:Fixnum[3] = Const Value(3) + v87:Fixnum[4] = Const Value(4) + v102:ArrayExact = NewArray v57, v67, v77, v87 CheckInterrupts PopInlineFrame v14:Fixnum[10] = Const Value(10) v16:Fixnum[20] = Const Value(20) v18:Fixnum[30] = Const Value(30) PushInlineFrame v44 (0x1038), v14, v16, v18 - v121:Fixnum[4] = Const Value(4) - v136:ArrayExact = NewArray v14, v16, v18, v121 + v123:Fixnum[4] = Const Value(4) + v138:ArrayExact = NewArray v14, v16, v18, v123 PopInlineFrame v24:Fixnum[10] = Const Value(10) v26:Fixnum[20] = Const Value(20) @@ -1882,7 +1882,7 @@ mod hir_opt_tests { v30:Fixnum[40] = Const Value(40) v32:Fixnum[50] = Const Value(50) v34:BasicObject = Send v44, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count - v37:ArrayExact = NewArray v101, v136, v34 + v37:ArrayExact = NewArray v102, v138, v34 CheckInterrupts Return v37 "); @@ -3797,10 +3797,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile PushInlineFrame v23 (0x1040) - v29:ArrayExact = NewArray + v30:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v29 + Return v30 "); } @@ -3828,25 +3828,25 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v51:NilClass = Const Value(nil) + v52:NilClass = Const Value(nil) PushInlineFrame v22 (0x1038), v11, v13 - v33:CPtr = GetEP 0 - v34:CUInt64 = LoadField v33, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v35:CBool = IsBlockParamModified v34 - CondBranch v35, bb6(), bb7() + v34:CPtr = GetEP 0 + v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v36:CBool = IsBlockParamModified v35 + CondBranch v36, bb6(), bb7() bb6(): - v37:BasicObject = LoadField v33, :block@0x1041 - Jump bb8(v37, v37) + v38:BasicObject = LoadField v34, :block@0x1041 + Jump bb8(v38, v38) bb7(): - v39:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v40:CInt64 = GuardAnyBitSet v39, CUInt64(1) recompile - v41:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) - Jump bb8(v41, v51) - bb8(v31:BasicObject, v32:BasicObject): - v46:BasicObject = Send v31, :call, v11, v13 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall + v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) recompile + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) + Jump bb8(v42, v52) + bb8(v32:BasicObject, v33:BasicObject): + v47:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -3986,7 +3986,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:Fixnum[80] = Const Value(80) + v28:Fixnum[80] = Const Value(80) CheckInterrupts PopInlineFrame Return v11 @@ -4018,12 +4018,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 - v30:Fixnum[80] = Const Value(80) + v31:Fixnum[80] = Const Value(80) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v66:Fixnum[110] = Const Value(110) + v67:Fixnum[110] = Const Value(110) CheckInterrupts PopInlineFrame - Return v66 + Return v67 "); } @@ -4050,12 +4050,12 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1038), v11, v13 - v33:ArrayExact = NewArray v11, v13 + v34:ArrayExact = NewArray v11, v13 CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); } @@ -4083,12 +4083,12 @@ mod hir_opt_tests { v15:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1038), v13, v15, v11 - v38:ArrayExact = NewArray v13, v15, v11 + v39:ArrayExact = NewArray v13, v15, v11 CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -4116,12 +4116,12 @@ mod hir_opt_tests { v15:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1038), v11, v15, v13 - v38:ArrayExact = NewArray v11, v15, v13 + v39:ArrayExact = NewArray v11, v15, v13 CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -4148,12 +4148,12 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1038), v11, v13 - v33:ArrayExact = NewArray v11, v13 + v34:ArrayExact = NewArray v11, v13 CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); } @@ -4181,21 +4181,21 @@ mod hir_opt_tests { v15:Fixnum[4] = Const Value(4) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v71:Fixnum[0] = Const Value(0) + v72:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v11, v13, v15 - v51:Fixnum[2] = Const Value(2) - v65:ArrayExact = NewArray v51, v13 + v52:Fixnum[2] = Const Value(2) + v66:ArrayExact = NewArray v52, v13 CheckInterrupts PopInlineFrame v20:Fixnum[1] = Const Value(1) v22:Fixnum[2] = Const Value(2) v24:Fixnum[4] = Const Value(4) v26:Fixnum[3] = Const Value(3) - v92:Fixnum[0] = Const Value(0) + v94:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v20, v22, v26, v24 - v87:ArrayExact = NewArray v22, v26 + v89:ArrayExact = NewArray v22, v26 PopInlineFrame - v30:ArrayExact = NewArray v65, v87 + v30:ArrayExact = NewArray v66, v89 Return v30 "); } @@ -4224,21 +4224,21 @@ mod hir_opt_tests { v34:Fixnum[4] = Const Value(4) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v73:Fixnum[0] = Const Value(0) + v74:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v11, v13, v34 - v51:Fixnum[2] = Const Value(2) - v67:ArrayExact = NewArray v11, v51, v13, v34 + v52:Fixnum[2] = Const Value(2) + v68:ArrayExact = NewArray v11, v52, v13, v34 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) v20:Fixnum[2] = Const Value(2) v22:Fixnum[40] = Const Value(40) v24:Fixnum[30] = Const Value(30) - v96:Fixnum[0] = Const Value(0) + v98:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v18, v20, v24, v22 - v91:ArrayExact = NewArray v18, v20, v24, v22 + v93:ArrayExact = NewArray v18, v20, v24, v22 PopInlineFrame - v28:ArrayExact = NewArray v67, v91 + v28:ArrayExact = NewArray v68, v93 Return v28 "); } @@ -4307,7 +4307,7 @@ mod hir_opt_tests { v11:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v32:Fixnum[0] = Const Value(0) + v33:Fixnum[0] = Const Value(0) PushInlineFrame v20 (0x1038), v11 CheckInterrupts PopInlineFrame @@ -4424,14 +4424,14 @@ mod hir_opt_tests { v17:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v37:Fixnum[0] = Const Value(0) + v38:Fixnum[0] = Const Value(0) PushInlineFrame v20 (0x1038), v17 - v29:Fixnum[1] = Const Value(1) + v30:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v46:Fixnum[2] = Const Value(2) + v47:Fixnum[2] = Const Value(2) CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -4729,12 +4729,12 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(C@0x1008) PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v48 (0x1068), v17 - v65:CShape = LoadField v48, :shape_id@0x1070 - v66:CShape[0x1071] = GuardBitEquals v65, CShape(0x1071) recompile + v66:CShape = LoadField v48, :shape_id@0x1070 + v67:CShape[0x1071] = GuardBitEquals v66, CShape(0x1071) recompile StoreField v48, :@x@0x1072, v17 WriteBarrier v48, v17 - v69:CShape[0x1073] = Const CShape(0x1073) - StoreField v48, :shape_id@0x1070, v69 + v70:CShape[0x1073] = Const CShape(0x1073) + StoreField v48, :shape_id@0x1070, v70 CheckInterrupts PopInlineFrame Return v48 @@ -4829,22 +4829,22 @@ mod hir_opt_tests { v46:Fixnum[0] = Const Value(0) PatchPoint NoSingletonClass(Hash@0x1008) PatchPoint MethodRedefined(Hash@0x1008, initialize@0x1038, cme:0x1040) - v97:Fixnum[0] = Const Value(0) - v98:NilClass = Const Value(nil) + v98:Fixnum[0] = Const Value(0) + v99:NilClass = Const Value(nil) PushInlineFrame v45 (0x1068), v46 - v64:TrueClass = Const Value(true) - v82:CPtr = GetEP 0 - v83:CUInt64 = LoadField v82, :VM_ENV_DATA_INDEX_FLAGS@0x1070 - v84:CBool = IsBlockParamModified v83 - CondBranch v84, bb11(), bb12() + v65:TrueClass = Const Value(true) + v83:CPtr = GetEP 0 + v84:CUInt64 = LoadField v83, :VM_ENV_DATA_INDEX_FLAGS@0x1070 + v85:CBool = IsBlockParamModified v84 + CondBranch v85, bb11(), bb12() bb11(): - v86:BasicObject = LoadField v82, :block@0x1071 - Jump bb13(v86) + v87:BasicObject = LoadField v83, :block@0x1071 + Jump bb13(v87) bb12(): - v88:BasicObject = GetBlockParam :block, l0, EP@4 - Jump bb13(v88) - bb13(v81:BasicObject): - v91:BasicObject = InvokeBuiltin rb_hash_init, v45, v46, v64, v64, v81 + v89:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb13(v89) + bb13(v82:BasicObject): + v92:BasicObject = InvokeBuiltin rb_hash_init, v45, v46, v65, v65, v82 CheckInterrupts PopInlineFrame Return v45 @@ -7334,10 +7334,10 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) PushInlineFrame v11 (0x1040), v13 - v30:ArrayExact = NewArray + v31:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v30 + Return v31 "); } @@ -7396,10 +7396,10 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) PushInlineFrame v11 (0x1040) - v26:ArrayExact = NewArray + v27:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v26 + Return v27 "); } @@ -9622,11 +9622,11 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v18 (0x1038) - v24:Fixnum[1] = Const Value(1) - v26:BasicObject = InvokeBlock v24 # SendFallbackReason: InvokeBlock: not yet specialized + v25:Fixnum[1] = Const Value(1) + v27:BasicObject = InvokeBlock v25 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts PopInlineFrame - Return v26 + Return v27 "); } @@ -9655,49 +9655,49 @@ mod hir_opt_tests { bb3(v6:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v71:NilClass = Const Value(nil) + v72:NilClass = Const Value(nil) PushInlineFrame v18 (0x1038) - v27:CPtr = GetEP 0 - v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v29:CBool = IsBlockParamModified v28 - CondBranch v29, bb7(), bb8() + v28:CPtr = GetEP 0 + v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v30:CBool = IsBlockParamModified v29 + CondBranch v30, bb7(), bb8() bb7(): - v31:BasicObject = LoadField v27, :blk@0x1041 - Jump bb9(v31, v31) + v32:BasicObject = LoadField v28, :blk@0x1041 + Jump bb9(v32, v32) bb8(): - v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v34:CInt64[0] = GuardBitEquals v33, CInt64(0) recompile - v35:NilClass = Const Value(nil) - Jump bb9(v35, v71) - bb9(v25:BasicObject, v26:BasicObject): + v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile + v36:NilClass = Const Value(nil) + Jump bb9(v36, v72) + bb9(v26:BasicObject, v27:BasicObject): CheckInterrupts - v39:CBool = Test v25 - CondBranch v39, bb10(), bb6(v18, v26) + v40:CBool = Test v26 + CondBranch v40, bb10(), bb6(v18, v27) bb10(): - v46:CPtr = GetEP 0 - v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v48:CBool = IsBlockParamModified v47 - CondBranch v48, bb11(), bb12() + v47:CPtr = GetEP 0 + v48:CUInt64 = LoadField v47, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v49:CBool = IsBlockParamModified v48 + CondBranch v49, bb11(), bb12() bb11(): - v50:BasicObject = LoadField v46, :blk@0x1041 - Jump bb13(v50, v50) + v51:BasicObject = LoadField v47, :blk@0x1041 + Jump bb13(v51, v51) bb12(): - v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v53:CInt64 = GuardAnyBitSet v52, CUInt64(1) recompile - v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) - Jump bb13(v54, v26) - bb13(v44:BasicObject, v45:BasicObject): - v57:BasicObject = Send v44, :call # SendFallbackReason: SendWithoutBlock: no profile data available - CheckInterrupts - Jump bb4(v57) - bb6(v62:ObjectSubclass[class_exact*:Object@VALUE(0x1000)], v63:BasicObject): - v66:Fixnum[42] = Const Value(42) - CheckInterrupts - Jump bb4(v66) - bb4(v72:BasicObject): + v53:CInt64 = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v54:CInt64 = GuardAnyBitSet v53, CUInt64(1) recompile + v55:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) + Jump bb13(v55, v27) + bb13(v45:BasicObject, v46:BasicObject): + v58:BasicObject = Send v45, :call # SendFallbackReason: SendWithoutBlock: no profile data available + CheckInterrupts + Jump bb4(v58) + bb6(v63:ObjectSubclass[class_exact*:Object@VALUE(0x1000)], v64:BasicObject): + v67:Fixnum[42] = Const Value(42) + CheckInterrupts + Jump bb4(v67) + bb4(v73:BasicObject): PopInlineFrame CheckInterrupts - Return v72 + Return v73 "); } @@ -9726,12 +9726,12 @@ mod hir_opt_tests { PatchPoint SingleRactorMode PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v48:NilClass = Const Value(nil) + v49:NilClass = Const Value(nil) PushInlineFrame v19 (0x1038) CheckInterrupts - v43:Fixnum[42] = Const Value(42) + v44:Fixnum[42] = Const Value(42) PopInlineFrame - Return v43 + Return v44 "); } @@ -15508,11 +15508,11 @@ mod hir_opt_tests { v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 v22:FalseClass = GuardBitEquals v21, Value(false) PushInlineFrame v6 (0x1050) - v28:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) - v29:StringExact = StringCopy v28 + v29:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) + v30:StringExact = StringCopy v29 CheckInterrupts PopInlineFrame - Return v29 + Return v30 "); } @@ -15582,15 +15582,15 @@ mod hir_opt_tests { v31:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 v32:FalseClass = GuardBitEquals v31, Value(false) PushInlineFrame v9 (0x1058), v10 - v44:Fixnum[2] = Const Value(2) + v45:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) - v58:Fixnum = GuardType v10, Fixnum recompile - v59:Fixnum = FixnumMult v58, v44 + v59:Fixnum = GuardType v10, Fixnum recompile + v60:Fixnum = FixnumMult v59, v45 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) - v37:Fixnum = FixnumAdd v59, v18 + v37:Fixnum = FixnumAdd v60, v18 Return v37 "); } @@ -16938,10 +16938,10 @@ mod hir_opt_tests { v43:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v43 (0x1040), v23 PatchPoint MethodRedefined(Integer@0x1048, to_s@0x1050, cme:0x1058) - v63:StringExact = CCallVariadic v23, :Integer#to_s@0x1080 + v64:StringExact = CCallVariadic v23, :Integer#to_s@0x1080 CheckInterrupts PopInlineFrame - Return v63 + Return v64 bb4(v30:BasicObject, v31:Falsy): v35:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) v36:StringExact = StringCopy v35 @@ -18051,11 +18051,11 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v44:Fixnum = GuardType v10, Fixnum recompile - v46:Fixnum = FixnumAdd v44, v44 + v45:Fixnum = GuardType v10, Fixnum recompile + v47:Fixnum = FixnumAdd v45, v45 CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -18092,24 +18092,24 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, clamp_nonneg@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[0] = Const Value(0) + v32:Fixnum[0] = Const Value(0) PatchPoint MethodRedefined(Integer@0x1048, <@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:BoolExact = FixnumLt v62, v31 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:BoolExact = FixnumLt v63, v32 CheckInterrupts - v37:CBool = Test v63 - CondBranch v37, bb7(), bb6(v23, v62) + v38:CBool = Test v64 + CondBranch v38, bb7(), bb6(v23, v63) bb7(): - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) CheckInterrupts - Jump bb4(v42) - bb6(v47:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v48:Fixnum): + Jump bb4(v43) + bb6(v48:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v49:Fixnum): CheckInterrupts - Jump bb4(v48) - bb4(v56:Fixnum): + Jump bb4(v49) + bb4(v57:Fixnum): PopInlineFrame CheckInterrupts - Return v56 + Return v57 "); } @@ -18142,13 +18142,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_one@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[1] = Const Value(1) + v32:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v45:Fixnum = GuardType v10, Fixnum recompile - v46:Fixnum = FixnumAdd v45, v31 + v46:Fixnum = GuardType v10, Fixnum recompile + v47:Fixnum = FixnumAdd v46, v32 CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -18205,10 +18205,10 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 PatchPoint MethodRedefined(Object@0x1008, inner@0x1048, cme:0x1050) - v43:BasicObject = SendDirect v23, 0x1078, :inner (0x1088), v10 + v44:BasicObject = SendDirect v23, 0x1078, :inner (0x1088), v10 CheckInterrupts PopInlineFrame - Return v43 + Return v44 "); } @@ -18306,15 +18306,15 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[10] = Const Value(10) - v40:Fixnum[100] = Const Value(100) + v32:Fixnum[10] = Const Value(10) + v41:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v70:Fixnum = GuardType v10, Fixnum recompile - v71:Fixnum = FixnumAdd v70, v31 - v75:Fixnum = FixnumAdd v71, v40 + v71:Fixnum = GuardType v10, Fixnum recompile + v72:Fixnum = FixnumAdd v71, v32 + v76:Fixnum = FixnumAdd v72, v41 CheckInterrupts PopInlineFrame - Return v75 + Return v76 "); } @@ -18363,14 +18363,14 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10, v16 - v33:Fixnum[100] = Const Value(100) + v34:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:Fixnum = FixnumAdd v62, v16 - v67:Fixnum = FixnumAdd v63, v33 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:Fixnum = FixnumAdd v63, v16 + v68:Fixnum = FixnumAdd v64, v34 CheckInterrupts PopInlineFrame - Return v67 + Return v68 "); } @@ -18419,13 +18419,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, maybe_rescue@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[1] = Const Value(1) + v32:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v46:Fixnum = GuardType v10, Fixnum recompile - v47:Fixnum = FixnumAdd v46, v31 + v47:Fixnum = GuardType v10, Fixnum recompile + v48:Fixnum = FixnumAdd v47, v32 CheckInterrupts PopInlineFrame - Return v47 + Return v48 "); } @@ -18518,24 +18518,24 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact:Child] = GuardType v10, ObjectSubclass[class_exact:Child] recompile PushInlineFrame v23 (0x1040) PatchPoint MethodRedefined(Parent@0x1048, greet@0x1010, cme:0x1050) - v46:CPtr = GetEP 0 - v47:RubyValue = LoadField v46, :VM_ENV_DATA_INDEX_ME_CREF@0x1078 - v48:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v47, Value(VALUE(0x1018)) - v49:RubyValue = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1079 - v50:FalseClass = GuardBitEquals v49, Value(false) + v47:CPtr = GetEP 0 + v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1078 + v49:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v48, Value(VALUE(0x1018)) + v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1079 + v51:FalseClass = GuardBitEquals v50, Value(false) PushInlineFrame v23 (0x1040) - v61:StringExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) - v62:StringExact = StringCopy v61 + v63:StringExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) + v64:StringExact = StringCopy v63 CheckInterrupts PopInlineFrame - v32:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) - v33:StringExact = StringCopy v32 + v33:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) + v34:StringExact = StringCopy v33 PatchPoint NoSingletonClass(String@0x1090) PatchPoint MethodRedefined(String@0x1090, +@0x1098, cme:0x10a0) - v56:BasicObject = CCallWithFrame v62, :String#+@0x10c8, v33 + v57:BasicObject = CCallWithFrame v64, :String#+@0x10c8, v34 CheckInterrupts PopInlineFrame - Return v56 + Return v57 "); } @@ -18585,12 +18585,12 @@ mod hir_opt_tests { v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v27 (0x1040), v10, v16, v18 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v54:Fixnum = GuardType v10, Fixnum recompile - v55:Fixnum = FixnumAdd v54, v16 - v59:Fixnum = FixnumAdd v55, v18 + v55:Fixnum = GuardType v10, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v16 + v60:Fixnum = FixnumAdd v56, v18 CheckInterrupts PopInlineFrame - Return v59 + Return v60 "); } @@ -18639,13 +18639,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opt_post@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v30:Fixnum[10] = Const Value(10) + v31:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v55:Fixnum = GuardType v10, Fixnum - v56:Fixnum = FixnumAdd v30, v55 + v56:Fixnum = GuardType v10, Fixnum + v57:Fixnum = FixnumAdd v31, v56 CheckInterrupts PopInlineFrame - Return v56 + Return v57 "); } @@ -18695,14 +18695,14 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_lead_opt_post@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10, v16 - v33:Fixnum[10] = Const Value(10) + v34:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:Fixnum = FixnumAdd v62, v33 - v67:Fixnum = FixnumAdd v63, v16 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:Fixnum = FixnumAdd v63, v34 + v68:Fixnum = FixnumAdd v64, v16 CheckInterrupts PopInlineFrame - Return v67 + Return v68 "); } @@ -18747,14 +18747,14 @@ mod hir_opt_tests { v16:Fixnum[5] = Const Value(5) PatchPoint MethodRedefined(Object@0x1008, add_kw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v16 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v16 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18799,14 +18799,14 @@ mod hir_opt_tests { v16:Fixnum[50] = Const Value(50) PatchPoint MethodRedefined(Object@0x1008, add_optkw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v16 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v16 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18851,14 +18851,14 @@ mod hir_opt_tests { v22:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Object@0x1008, add_optkw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v22 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v22 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v22 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18905,19 +18905,19 @@ mod hir_opt_tests { v18:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1008, add_kws@0x1010, cme:0x1018) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v60:Fixnum[0] = Const Value(0) + v61:Fixnum[0] = Const Value(0) PushInlineFrame v28 (0x1040), v10, v18, v16 - v39:Fixnum[100] = Const Value(100) + v40:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, *@0x1050, cme:0x1058) - v67:Fixnum = GuardType v10, Fixnum recompile - v68:Fixnum = FixnumMult v67, v39 - v81:Fixnum[20] = Const Value(20) + v68:Fixnum = GuardType v10, Fixnum recompile + v69:Fixnum = FixnumMult v68, v40 + v82:Fixnum[20] = Const Value(20) PatchPoint MethodRedefined(Integer@0x1048, +@0x1080, cme:0x1088) - v76:Fixnum = FixnumAdd v68, v81 - v80:Fixnum = FixnumAdd v76, v16 + v77:Fixnum = FixnumAdd v69, v82 + v81:Fixnum = FixnumAdd v77, v16 CheckInterrupts PopInlineFrame - Return v80 + Return v81 "); } @@ -18963,26 +18963,26 @@ mod hir_opt_tests { v22:NilClass = Const Value(nil) PatchPoint MethodRedefined(Object@0x1008, add_optkw_dyn@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v63:Fixnum[1] = Const Value(1) + v64:Fixnum[1] = Const Value(1) PushInlineFrame v25 (0x1040), v10, v22 - v33:BoolExact = FixnumBitCheck v63, 0 + v34:BoolExact = FixnumBitCheck v64, 0 CheckInterrupts - v36:CBool = Test v33 - CondBranch v36, bb6(v25, v10, v22, v63), bb7() + v37:CBool = Test v34 + CondBranch v37, bb6(v25, v10, v22, v64), bb7() bb7(): - v42:Fixnum[2] = Const Value(2) + v43:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1048, *@0x1050, cme:0x1058) - v70:Fixnum = GuardType v10, Fixnum recompile - v71:Fixnum = FixnumMult v70, v42 - Jump bb6(v25, v70, v71, v63) - bb6(v48:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v49:BasicObject, v50:NilClass|Fixnum, v51:Fixnum[1]): + v71:Fixnum = GuardType v10, Fixnum recompile + v72:Fixnum = FixnumMult v71, v43 + Jump bb6(v25, v71, v72, v64) + bb6(v49:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v50:BasicObject, v51:NilClass|Fixnum, v52:Fixnum[1]): PatchPoint MethodRedefined(Integer@0x1048, +@0x1080, cme:0x1088) - v74:Fixnum = GuardType v49, Fixnum recompile - v75:Fixnum = GuardType v50, Fixnum - v76:Fixnum = FixnumAdd v74, v75 + v75:Fixnum = GuardType v50, Fixnum recompile + v76:Fixnum = GuardType v51, Fixnum + v77:Fixnum = FixnumAdd v75, v76 CheckInterrupts PopInlineFrame - Return v76 + Return v77 "); } @@ -19033,12 +19033,12 @@ mod hir_opt_tests { v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v27 (0x1040), v10, v16, v18 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v54:Fixnum = GuardType v10, Fixnum recompile - v55:Fixnum = FixnumAdd v54, v16 - v59:Fixnum = FixnumAdd v55, v18 + v55:Fixnum = GuardType v10, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v16 + v60:Fixnum = FixnumAdd v56, v18 CheckInterrupts PopInlineFrame - Return v59 + Return v60 "); } @@ -19087,11 +19087,11 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, with_yield@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10 - v33:BasicObject = InvokeBlock v10 # SendFallbackReason: InvokeBlock: not yet specialized + v34:BasicObject = InvokeBlock v10 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v33 + Return v34 "); } @@ -19138,26 +19138,26 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, with_block_param@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v52:NilClass = Const Value(nil) + v53:NilClass = Const Value(nil) PushInlineFrame v25 (0x1040), v10 - v35:CPtr = GetEP 0 - v36:CUInt64 = LoadField v35, :VM_ENV_DATA_INDEX_FLAGS@0x1048 - v37:CBool = IsBlockParamModified v36 - CondBranch v37, bb6(), bb7() + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1048 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb6(), bb7() bb6(): - v39:BasicObject = LoadField v35, :block@0x1049 - Jump bb8(v39, v39) + v40:BasicObject = LoadField v36, :block@0x1049 + Jump bb8(v40, v40) bb7(): - v41:CInt64 = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x104a - v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) recompile - v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) - Jump bb8(v43, v52) - bb8(v33:BasicObject, v34:BasicObject): - v47:BasicObject = Send v33, :call, v10 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x104a + v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) + Jump bb8(v44, v53) + bb8(v34:BasicObject, v35:BasicObject): + v48:BasicObject = Send v34, :call, v10 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v47 + Return v48 "); } @@ -19202,26 +19202,26 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, callee@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v53:NilClass = Const Value(nil) + v54:NilClass = Const Value(nil) PushInlineFrame v25 (0x1040), v10 - v37:CPtr = GetEP 0 - v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1048 - v39:CBool = IsBlockParamModified v38 - CondBranch v39, bb6(), bb7() + v38:CPtr = GetEP 0 + v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1048 + v40:CBool = IsBlockParamModified v39 + CondBranch v40, bb6(), bb7() bb6(): - v41:BasicObject = LoadField v37, :block@0x1049 - Jump bb8(v41, v41) + v42:BasicObject = LoadField v38, :block@0x1049 + Jump bb8(v42, v42) bb7(): - v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x104a - v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) recompile - v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) - Jump bb8(v45, v53) - bb8(v35:BasicObject, v36:BasicObject): - v48:BasicObject = Send v25, &block, :inner, v10, v35 # SendFallbackReason: Complex argument passing + v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x104a + v45:CInt64 = GuardAnyBitSet v44, CUInt64(1) recompile + v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) + Jump bb8(v46, v54) + bb8(v36:BasicObject, v37:BasicObject): + v49:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Complex argument passing CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v48 + Return v49 "); } @@ -19281,18 +19281,18 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v91 (0x1068), v17, v19 - v122:CShape = LoadField v91, :shape_id@0x1070 - v123:CShape[0x1071] = GuardBitEquals v122, CShape(0x1071) recompile + v123:CShape = LoadField v91, :shape_id@0x1070 + v124:CShape[0x1071] = GuardBitEquals v123, CShape(0x1071) recompile StoreField v91, :@x@0x1072, v17 WriteBarrier v91, v17 - v126:CShape[0x1073] = Const CShape(0x1073) - StoreField v91, :shape_id@0x1070, v126 + v127:CShape[0x1073] = Const CShape(0x1073) + StoreField v91, :shape_id@0x1070, v127 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode StoreField v91, :@y@0x1074, v19 WriteBarrier v91, v19 - v141:CShape[0x1075] = Const CShape(0x1075) - StoreField v91, :shape_id@0x1070, v141 + v142:CShape[0x1075] = Const CShape(0x1075) + StoreField v91, :shape_id@0x1070, v142 CheckInterrupts PopInlineFrame PatchPoint SingleRactorMode @@ -19306,55 +19306,55 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v101 (0x1068), v52, v54 - v161:CShape = LoadField v101, :shape_id@0x1070 - v162:CShape[0x1071] = GuardBitEquals v161, CShape(0x1071) recompile + v163:CShape = LoadField v101, :shape_id@0x1070 + v164:CShape[0x1071] = GuardBitEquals v163, CShape(0x1071) recompile StoreField v101, :@x@0x1072, v52 WriteBarrier v101, v52 - v165:CShape[0x1073] = Const CShape(0x1073) - StoreField v101, :shape_id@0x1070, v165 + v167:CShape[0x1073] = Const CShape(0x1073) + StoreField v101, :shape_id@0x1070, v167 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode StoreField v101, :@y@0x1074, v54 WriteBarrier v101, v54 - v180:CShape[0x1075] = Const CShape(0x1075) - StoreField v101, :shape_id@0x1070, v180 + v182:CShape[0x1075] = Const CShape(0x1075) + StoreField v101, :shape_id@0x1070, v182 CheckInterrupts PopInlineFrame PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, ==@0x1080, cme:0x1088) PushInlineFrame v91 (0x1068), v101 PatchPoint SingleRactorMode - v198:CShape = LoadField v91, :shape_id@0x1070 - v199:CShape[0x1075] = GuardBitEquals v198, CShape(0x1075) recompile - v200:BasicObject = LoadField v91, :@x@0x1072 + v201:CShape = LoadField v91, :shape_id@0x1070 + v202:CShape[0x1075] = GuardBitEquals v201, CShape(0x1075) recompile + v203:BasicObject = LoadField v91, :@x@0x1072 PatchPoint NoEPEscape(==) PatchPoint MethodRedefined(Point@0x1008, x@0x10b0, cme:0x10b8) PatchPoint MethodRedefined(Integer@0x10e0, ==@0x1080, cme:0x10e8) - v255:Fixnum = GuardType v200, Fixnum recompile - v257:BoolExact = FixnumEq v255, v52 - v212:CBool = Test v257 - v213:FalseClass = RefineType v257, Falsy - CondBranch v212, bb19(), bb18(v91, v101, v213) + v258:Fixnum = GuardType v203, Fixnum recompile + v260:BoolExact = FixnumEq v258, v52 + v215:CBool = Test v260 + v216:FalseClass = RefineType v260, Falsy + CondBranch v215, bb19(), bb18(v91, v101, v216) bb19(): PatchPoint SingleRactorMode - v220:CShape = LoadField v91, :shape_id@0x1070 - v221:CShape[0x1075] = GuardBitEquals v220, CShape(0x1075) recompile - v222:BasicObject = LoadField v91, :@y@0x1074 + v223:CShape = LoadField v91, :shape_id@0x1070 + v224:CShape[0x1075] = GuardBitEquals v223, CShape(0x1075) recompile + v225:BasicObject = LoadField v91, :@y@0x1074 PatchPoint NoEPEscape(==) PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, y@0x1110, cme:0x1118) - v262:CShape = LoadField v101, :shape_id@0x1070 - v263:CShape[0x1075] = GuardBitEquals v262, CShape(0x1075) recompile - v264:BasicObject = LoadField v101, :@y@0x1074 + v265:CShape = LoadField v101, :shape_id@0x1070 + v266:CShape[0x1075] = GuardBitEquals v265, CShape(0x1075) recompile + v267:BasicObject = LoadField v101, :@y@0x1074 PatchPoint MethodRedefined(Integer@0x10e0, ==@0x1080, cme:0x10e8) - v267:Fixnum = GuardType v222, Fixnum recompile - v268:Fixnum = GuardType v264, Fixnum - v269:BoolExact = FixnumEq v267, v268 - Jump bb18(v91, v101, v269) - bb18(v232:ObjectSubclass[class_exact:Point], v233:ObjectSubclass[class_exact:Point], v234:BoolExact): + v270:Fixnum = GuardType v225, Fixnum recompile + v271:Fixnum = GuardType v267, Fixnum + v272:BoolExact = FixnumEq v270, v271 + Jump bb18(v91, v101, v272) + bb18(v235:ObjectSubclass[class_exact:Point], v236:ObjectSubclass[class_exact:Point], v237:BoolExact): CheckInterrupts PopInlineFrame - Return v234 + Return v237 "); } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index a61aa4c11bb7a3..02de128ef4ee32 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -136,14 +136,14 @@ mod snapshot_tests { v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1048), v13, v15, v11 - v37:Any = Snapshot FrameState { pc: 0x1050, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v43], caller: v23 } - v38:ArrayExact = NewArray v13, v15, v11 - v39:Any = Snapshot FrameState { pc: 0x1058, stack: [v38], locals: [a=v13, b=v15, c=v11, ID(0)=v43], caller: v23 } + v38:Any = Snapshot FrameState { pc: 0x1050, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v39:ArrayExact = NewArray v13, v15, v11 + v40:Any = Snapshot FrameState { pc: 0x1058, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -175,14 +175,14 @@ mod snapshot_tests { v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1048), v11, v13 - v32:Any = Snapshot FrameState { pc: 0x1050, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v38], caller: v14 } - v33:ArrayExact = NewArray v11, v13 - v34:Any = Snapshot FrameState { pc: 0x1058, stack: [v33], locals: [a=v11, b=v13, ID(0)=v38], caller: v14 } + v33:Any = Snapshot FrameState { pc: 0x1050, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v34:ArrayExact = NewArray v11, v13 + v35:Any = Snapshot FrameState { pc: 0x1058, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); } From df0622320bbd9a7c345e36e1b0730f25bc6a016b Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Tue, 30 Jun 2026 15:48:34 +0900 Subject: [PATCH 5/8] [Feature #22135] Remove obsolete `ObjectSpace._id2ref` --- NEWS.md | 4 + bootstraptest/test_objectspace.rb | 24 -- bootstraptest/test_ractor.rb | 44 --- gc.c | 300 +-------------------- gc/gc.h | 1 - internal/gc.h | 1 - ractor.c | 2 - spec/ruby/core/objectspace/_id2ref_spec.rb | 2 +- test/ruby/test_objectspace.rb | 86 ------ 9 files changed, 7 insertions(+), 457 deletions(-) diff --git a/NEWS.md b/NEWS.md index c666338ffafbe0..0d725703b9c9f9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -43,6 +43,10 @@ Note: We're only listing outstanding class updates. * `MatchData#integer_at` is added. It converts the matched substring to integer and return the result. [[Feature #21932]] +* ObjectSpace + + * `ObjectSpace._id2ref` was removed. [[Feature #22135]] + * Regexp * All instances of `Regexp` are now frozen, not just literals. diff --git a/bootstraptest/test_objectspace.rb b/bootstraptest/test_objectspace.rb index 63a8d9932287f6..bde5e9c0b409e7 100644 --- a/bootstraptest/test_objectspace.rb +++ b/bootstraptest/test_objectspace.rb @@ -1,18 +1,3 @@ -assert_normal_exit %q{ - eval("", TOPLEVEL_BINDING) - minobj = ObjectSpace.to_enum(:each_object).min_by {|a| a.object_id } - maxobj = ObjectSpace.to_enum(:each_object).max_by {|a| a.object_id } - (((minobj.object_id-100)..(minobj.object_id+100))+ - ((maxobj.object_id-100)..(maxobj.object_id+100))).each {|id| - begin - o = ObjectSpace._id2ref(id) - rescue RangeError - next - end - o.inspect if defined?(o.inspect) - } -}, '[ruby-dev:31911]' - assert_normal_exit %q{ ary = (1..10).to_a ary.permutation(2) {|x| @@ -44,12 +29,3 @@ Thread.new {} end }, '[ruby-core:37858]' - -assert_equal 'ok', %q{ - objects_and_ids = 1000.times.map { o = Object.new; [o, o.object_id] } - objects_and_ids.each { |expected, id| - actual = ObjectSpace._id2ref(id) - raise "expected #{expected.inspect}, got #{actual.inspect}" unless actual.equal?(expected) - } - 'ok' -} diff --git a/bootstraptest/test_ractor.rb b/bootstraptest/test_ractor.rb index 4fe90703fc4c54..22d2ee47deb1ce 100644 --- a/bootstraptest/test_ractor.rb +++ b/bootstraptest/test_ractor.rb @@ -1256,37 +1256,6 @@ class C }.value } -# ObjectSpace._id2ref can not handle unshareable objects with Ractors -assert_equal 'ok', <<~'RUBY', frozen_string_literal: false - s = 'hello' - - Ractor.new s.object_id do |id ;s| - begin - s = ObjectSpace._id2ref(id) - rescue => e - :ok - end - end.value -RUBY - -# Inserting into the id2ref table should be Ractor-safe -assert_equal 'ok', <<~'RUBY' - # Force all calls to Kernel#object_id to insert into the id2ref table - obj = Object.new - ObjectSpace._id2ref(obj.object_id) rescue nil - - 10.times.map do - Ractor.new do - 10_000.times do - a = Object.new - a.object_id - end - end - end.map(&:value) - - :ok -RUBY - # Ractor.make_shareable(obj) assert_equal 'true', <<~'RUBY', frozen_string_literal: false class C @@ -2175,19 +2144,6 @@ def ==(o) roundtripped_obj.instance_variable_get(:@array1) == [1] ? :ok : roundtripped_obj } -# move object with generic ivars and existing id2ref table -# [Bug #21664] -assert_equal 'ok', %q{ - obj = [1] - obj.instance_variable_set("@field", :ok) - ObjectSpace._id2ref(obj.object_id) # build id2ref table - - ractor = Ractor.new { Ractor.receive } - ractor.send(obj, move: true) - obj = ractor.value - obj.instance_variable_get("@field") -} - # copy object with complex generic ivars assert_equal 'ok', %q{ # Make Array complex diff --git a/gc.c b/gc.c index d12e37f4346bed..e521369caa72ef 100644 --- a/gc.c +++ b/gc.c @@ -586,7 +586,7 @@ rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val) #endif static const char *obj_type_name(VALUE obj); -static st_table *id2ref_tbl; + #include "gc/default/default.c" #if USE_MODULAR_GC && !defined(HAVE_DLOPEN) @@ -1353,7 +1353,7 @@ rb_gc_imemo_needs_cleanup_p(VALUE obj) return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL; case imemo_fields: - return FL_TEST_RAW(obj, OBJ_FIELD_HEAP) || (id2ref_tbl && rb_obj_shape_has_id(obj)); + return FL_TEST_RAW(obj, OBJ_FIELD_HEAP); } UNREACHABLE_RETURN(true); } @@ -1365,7 +1365,6 @@ rb_gc_imemo_needs_cleanup_p(VALUE obj) * Objects that return false are: * - Simple embedded objects without external allocations * - Objects without finalizers - * - Objects without object IDs registered in id2ref * - Objects without generic instance variables * * This is used by the GC sweep fast path to avoid function call overhead @@ -1405,7 +1404,6 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) } shape_id_t shape_id = RBASIC_SHAPE_ID(obj); - if (id2ref_tbl && rb_shape_has_object_id(shape_id)) return true; switch (flags & RUBY_T_MASK) { case T_OBJECT: @@ -2054,7 +2052,6 @@ rb_objspace_garbage_object_p(VALUE obj) #define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1) #define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT) -static VALUE id2ref_value = 0; #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG static size_t object_id_counter = 1; @@ -2076,76 +2073,8 @@ generate_next_object_id(void) #endif } -void -rb_gc_obj_id_moved(VALUE obj) -{ - if (UNLIKELY(id2ref_tbl)) { - st_insert(id2ref_tbl, (st_data_t)rb_obj_id(obj), (st_data_t)obj); - } -} - -static int -object_id_cmp(st_data_t x, st_data_t y) -{ - if (RB_TYPE_P(x, T_BIGNUM)) { - return !rb_big_eql(x, y); - } - else { - return x != y; - } -} - -static st_index_t -object_id_hash(st_data_t n) -{ - return FIX2LONG(rb_hash((VALUE)n)); -} - -static const struct st_hash_type object_id_hash_type = { - object_id_cmp, - object_id_hash, -}; - static void gc_mark_tbl_no_pin(st_table *table); -static void -id2ref_tbl_mark(void *data) -{ - st_table *table = (st_table *)data; - if (UNLIKELY(!RB_POSFIXABLE(LAST_OBJECT_ID()))) { - // It's very unlikely, but if enough object ids were generated, keys may be T_BIGNUM - rb_mark_set(table); - } - // We purposely don't mark values, as they are weak references. - // rb_gc_obj_free_vm_weak_references takes care of cleaning them up. -} - -static size_t -id2ref_tbl_memsize(const void *data) -{ - return rb_st_memsize(data); -} - -static void -id2ref_tbl_free(void *data) -{ - id2ref_tbl = NULL; // clear global ref - st_table *table = (st_table *)data; - st_free_table(table); -} - -static const rb_data_type_t id2ref_tbl_type = { - .wrap_struct_name = "VM/_id2ref_table", - .function = { - .dmark = id2ref_tbl_mark, - .dfree = id2ref_tbl_free, - .dsize = id2ref_tbl_memsize, - // dcompact function not required because the table is reference updated - // in rb_gc_vm_weak_table_foreach - }, - .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY -}; - static VALUE class_object_id(VALUE klass) { @@ -2157,9 +2086,6 @@ class_object_id(VALUE klass) if (existing_id) { id = existing_id; } - else if (RB_UNLIKELY(id2ref_tbl)) { - st_insert(id2ref_tbl, id, klass); - } RB_GC_VM_UNLOCK(lock_lev); } return id; @@ -2203,11 +2129,6 @@ object_id0(VALUE obj) RUBY_ASSERT(rb_obj_shape_has_id(obj)); - if (RB_UNLIKELY(id2ref_tbl)) { - RB_VM_LOCKING() { - st_insert(id2ref_tbl, (st_data_t)id, (st_data_t)obj); - } - } return id; } @@ -2238,123 +2159,9 @@ object_id(VALUE obj) return object_id0(obj); } -static void -build_id2ref_i(VALUE obj, void *data) -{ - st_table *id2ref_tbl = (st_table *)data; - - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (RCLASS(obj)->object_id) { - st_insert(id2ref_tbl, RCLASS(obj)->object_id, obj); - } - break; - case T_IMEMO: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (IMEMO_TYPE_P(obj, imemo_fields) && rb_obj_shape_has_id(obj)) { - st_insert(id2ref_tbl, rb_obj_id(obj), rb_imemo_fields_owner(obj)); - } - break; - case T_OBJECT: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (rb_obj_shape_has_id(obj)) { - st_insert(id2ref_tbl, rb_obj_id(obj), obj); - } - break; - default: - // For generic_fields, the T_IMEMO/fields is responsible for populating the entry. - break; - } -} - -static VALUE -object_id_to_ref(void *objspace_ptr, VALUE object_id) -{ - rb_objspace_t *objspace = objspace_ptr; - - unsigned int lev = RB_GC_VM_LOCK(); - - if (!id2ref_tbl) { - rb_gc_vm_barrier(); // stop other ractors - - // GC Must not trigger while we build the table, otherwise if we end - // up freeing an object that had an ID, we might try to delete it from - // the table even though it wasn't inserted yet. - st_table *tmp_id2ref_tbl = st_init_table(&object_id_hash_type); - VALUE tmp_id2ref_value = TypedData_Wrap_Struct(0, &id2ref_tbl_type, tmp_id2ref_tbl); - - // build_id2ref_i will most certainly malloc, which could trigger GC and sweep - // objects we just added to the table. - // By calling rb_gc_disable() we also save having to handle potentially garbage objects. - bool gc_disabled = RTEST(rb_gc_disable()); - { - id2ref_tbl = tmp_id2ref_tbl; - id2ref_value = tmp_id2ref_value; - - rb_gc_impl_each_object(objspace, build_id2ref_i, (void *)id2ref_tbl); - } - if (!gc_disabled) rb_gc_enable(); - } - - VALUE obj; - bool found = st_lookup(id2ref_tbl, object_id, &obj) && !rb_gc_impl_garbage_object_p(objspace, obj); - - RB_GC_VM_UNLOCK(lev); - - if (found) { - return obj; - } - - if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(LAST_OBJECT_ID()))) { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } - else { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is a recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } -} - static inline void obj_free_object_id(VALUE obj) { - VALUE obj_id = 0; - if (RB_UNLIKELY(id2ref_tbl)) { - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - obj_id = RCLASS(obj)->object_id; - break; - case T_IMEMO: - if (!IMEMO_TYPE_P(obj, imemo_fields)) { - return; - } - // fallthrough - case T_OBJECT: - { - shape_id_t shape_id = RBASIC_SHAPE_ID(obj); - if (rb_shape_has_object_id(shape_id)) { - obj_id = object_id_get(obj, shape_id); - } - break; - } - default: - // For generic_fields, the T_IMEMO/fields is responsible for freeing the id. - return; - } - - if (RB_UNLIKELY(obj_id)) { - RUBY_ASSERT(FIXNUM_P(obj_id) || RB_TYPE_P(obj_id, T_BIGNUM)); - - if (!st_delete(id2ref_tbl, (st_data_t *)&obj_id, NULL)) { - // The the object is a T_IMEMO/fields, then it's possible the actual object - // has been garbage collected already. - if (!RB_TYPE_P(obj, T_IMEMO)) { - rb_bug("Object ID seen, but not in _id2ref table: object_id=%llu object=%s", NUM2ULL(obj_id), rb_obj_info(obj)); - } - } - } - } } void @@ -2393,67 +2200,6 @@ rb_gc_obj_free_vm_weak_references(VALUE obj) } } -/* - * call-seq: - * ObjectSpace._id2ref(object_id) -> an_object - * - * Converts an object id to a reference to the object. May not be - * called on an object id passed as a parameter to a finalizer. - * - * s = "I am a string" #=> "I am a string" - * r = ObjectSpace._id2ref(s.object_id) #=> "I am a string" - * r == s #=> true - * - * On multi-ractor mode, if the object is not shareable, it raises - * RangeError. - * - * This method is deprecated and should no longer be used. - */ - -static VALUE -id2ref(VALUE objid) -{ - objid = rb_to_int(objid); - if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) { - VALUE ptr = (VALUE)NUM2PTR(objid); - if (SPECIAL_CONST_P(ptr)) { - if (ptr == Qtrue) return Qtrue; - if (ptr == Qfalse) return Qfalse; - if (NIL_P(ptr)) return Qnil; - if (FIXNUM_P(ptr)) return ptr; - if (FLONUM_P(ptr)) return ptr; - - if (SYMBOL_P(ptr)) { - // Check that the symbol is valid - if (rb_static_id_valid_p(SYM2ID(ptr))) { - return ptr; - } - else { - rb_raise(rb_eRangeError, "%p is not a symbol id value", (void *)ptr); - } - } - - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_int2str(objid, 10)); - } - } - - VALUE obj = object_id_to_ref(rb_gc_get_objspace(), objid); - if (!rb_multi_ractor_p() || rb_ractor_shareable_p(obj)) { - return obj; - } - else { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is the id of an unshareable object on multi-ractor", rb_int2str(objid, 10)); - } -} - -/* :nodoc: */ -static VALUE -os_id2ref(VALUE os, VALUE objid) -{ - rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "ObjectSpace._id2ref is deprecated"); - return id2ref(objid); -} - static VALUE rb_find_object_id(void *objspace, VALUE obj, VALUE (*get_heap_object_id)(VALUE)) { @@ -4208,33 +3954,6 @@ vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data) struct st_table *rb_generic_fields_tbl_get(void); -static int -vm_weak_table_id2ref_foreach(st_data_t key, st_data_t value, st_data_t data, int error) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - if (!iter_data->weak_only && !FIXNUM_P((VALUE)key)) { - int ret = iter_data->callback((VALUE)key, iter_data->data); - if (ret != ST_CONTINUE) return ret; - } - - return iter_data->callback((VALUE)value, iter_data->data); -} - -static int -vm_weak_table_id2ref_foreach_update(st_data_t *key, st_data_t *value, st_data_t data, int existing) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - iter_data->update_callback((VALUE *)value, iter_data->data); - - if (!iter_data->weak_only && !FIXNUM_P((VALUE)*key)) { - iter_data->update_callback((VALUE *)key, iter_data->data); - } - - return ST_CONTINUE; -} - static int vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data) { @@ -4356,17 +4075,6 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, ); break; } - case RB_GC_VM_ID2REF_TABLE: { - if (id2ref_tbl) { - st_foreach_with_replace( - id2ref_tbl, - vm_weak_table_id2ref_foreach, - vm_weak_table_id2ref_foreach_update, - (st_data_t)&foreach_data - ); - } - break; - } case RB_GC_VM_GENERIC_FIELDS_TABLE: { st_table *generic_fields_tbl = rb_generic_fields_tbl_get(); if (generic_fields_tbl) { @@ -5870,8 +5578,6 @@ Init_GC(void) #endif #undef rb_intern - rb_gc_register_address(&id2ref_value); - malloc_offset = gc_compute_malloc_offset(); rb_mGC = rb_define_module("GC"); @@ -5883,8 +5589,6 @@ Init_GC(void) rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1); rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1); - rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1); - rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory"); rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0); diff --git a/gc/gc.h b/gc/gc.h index d8c351c7e92754..3de9b3f165ddb3 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -48,7 +48,6 @@ enum rb_gc_vm_weak_tables { RB_GC_VM_CI_TABLE, RB_GC_VM_OVERLOADED_CME_TABLE, RB_GC_VM_GLOBAL_SYMBOLS_TABLE, - RB_GC_VM_ID2REF_TABLE, RB_GC_VM_GENERIC_FIELDS_TABLE, RB_GC_VM_FROZEN_STRINGS_TABLE, RB_GC_VM_WEAK_TABLE_COUNT diff --git a/internal/gc.h b/internal/gc.h index 8a3dbd1884b4bf..9a137d2609554f 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -199,7 +199,6 @@ RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add(size_t, size_t, size_t); void *rb_xrealloc_mul_add(const void *, size_t, size_t, size_t); RUBY_ATTR_MALLOC void *rb_xmalloc_mul_add_mul(size_t, size_t, size_t, size_t); RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add_mul(size_t, size_t, size_t, size_t); -void rb_gc_obj_id_moved(VALUE obj); void rb_gc_register_pinning_obj(VALUE obj); rb_execution_context_t *rb_gc_get_ec(void); diff --git a/ractor.c b/ractor.c index fc6694c45c7f84..22b6432c4b1e81 100644 --- a/ractor.c +++ b/ractor.c @@ -2059,8 +2059,6 @@ move_leave(VALUE obj, struct obj_traverse_replace_data *data) rb_replace_generic_ivar(data->replacement, obj); } - rb_gc_obj_id_moved(data->replacement); - VALUE flags = T_OBJECT | FL_FREEZE | (RBASIC(obj)->flags & FL_PROMOTED); // Avoid mutations using bind_call, etc. diff --git a/spec/ruby/core/objectspace/_id2ref_spec.rb b/spec/ruby/core/objectspace/_id2ref_spec.rb index a9fd526b7d0f68..19461b8729f359 100644 --- a/spec/ruby/core/objectspace/_id2ref_spec.rb +++ b/spec/ruby/core/objectspace/_id2ref_spec.rb @@ -1,6 +1,6 @@ require_relative '../../spec_helper' -ruby_version_is "4.0" do +ruby_version_is "4.0"..."4.1" do describe "ObjectSpace._id2ref" do it "is deprecated" do id = nil.object_id diff --git a/test/ruby/test_objectspace.rb b/test/ruby/test_objectspace.rb index a479547599a03a..033ad0f0de56c0 100644 --- a/test/ruby/test_objectspace.rb +++ b/test/ruby/test_objectspace.rb @@ -2,77 +2,6 @@ require 'test/unit' class TestObjectSpace < Test::Unit::TestCase - def self.deftest_id2ref(obj) - /:(\d+)/ =~ caller[0] - file = $` - line = $1.to_i - code = <<"End" - define_method("test_id2ref_#{line}") {\ - o = EnvUtil.suppress_warning { ObjectSpace._id2ref(obj.object_id) } - assert_same(obj, o, "didn't round trip: \#{obj.inspect}");\ - } -End - eval code, binding, file, line - end - - deftest_id2ref(-0x4000000000000001) - deftest_id2ref(-0x4000000000000000) - deftest_id2ref(-0x40000001) - deftest_id2ref(-0x40000000) - deftest_id2ref(-1) - deftest_id2ref(0) - deftest_id2ref(1) - deftest_id2ref(0x3fffffff) - deftest_id2ref(0x40000000) - deftest_id2ref(0x3fffffffffffffff) - deftest_id2ref(0x4000000000000000) - deftest_id2ref(:a) - deftest_id2ref(:abcdefghijilkjl) - deftest_id2ref(:==) - deftest_id2ref(Object.new) - deftest_id2ref(self) - deftest_id2ref(true) - deftest_id2ref(false) - deftest_id2ref(nil) - - def test_id2ref_liveness - assert_normal_exit <<-EOS - ids = [] - 10.times{ - 1_000.times{ - ids << 'hello'.object_id - } - objs = ids.map{|id| - begin - ObjectSpace._id2ref(id) - rescue RangeError - nil - end - } - GC.start - objs.each{|e| e.inspect} - } - EOS - end - - def test_id2ref_invalid_argument - msg = /no implicit conversion/ - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(nil) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(false) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(true) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(:a) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref("0") } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(Object.new) } } - end - - def test_id2ref_invalid_symbol_id - # RB_STATIC_SYM_P checks for static symbols by checking that the bottom - # 8 bits of the object is equal to RUBY_SYMBOL_FLAG, so we need to make - # sure that the bottom 8 bits remain unchanged. - msg = /is not a symbol id value/ - assert_raise_with_message(RangeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(:a.object_id + 256) } } - end - def test_count_objects h = {} ObjectSpace.count_objects(h) @@ -284,21 +213,6 @@ def test_each_object_recursive_key end; end - def test_id2ref_table_build - assert_separately([], <<-End) - 10.times do - Object.new.object_id - end - - GC.start(immediate_mark: false) - - obj = Object.new - EnvUtil.suppress_warning do - assert_equal obj, ObjectSpace._id2ref(obj.object_id) - end - End - end - def test_each_object_singleton_class assert_separately([], <<-End) class C From d65b0ec0efaa488ceacb1c97350a373fe4e5ba9c Mon Sep 17 00:00:00 2001 From: Masataka Pocke Kuwabara Date: Thu, 9 Jul 2026 15:35:14 +0900 Subject: [PATCH 6/8] [ruby/json] Avoid re-decoding an incomplete number on every ResumableParser chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON::ResumableParser` re-decodes an incomplete number on every chunk it receives, so a long number delivered in small chunks is disproportionately slow to parse -- the work is quadratic (a bit worse in practice) in the number's length. ## Background When the parser reads a number that reaches the end of the buffer, it cannot tell whether more digits will arrive in a later chunk, so it rewinds the cursor and returns `false` to wait for more input. Until now it fully decoded the number before that check -- and then discarded the result. For a long run of digits that decode builds a bignum, whose cost grows faster than linearly with the digit count, and it was repeated on every chunk that extended the number. The total cost is therefore quadratic (a bit worse in practice) in the length of the number. The same "rewind and re-process" happens for incomplete strings and comments, but those only re-scan (no value construction), so only numbers pay this decode cost. ## Benchmark Feeding an incomplete token in 128-byte chunks, one iteration being one full chunked feed (`benchmark-ips`; macOS 15.6 arm64; ruby 4.0.0): ```ruby require "json" require "benchmark/ips" def feed_incomplete(token, chunk_size) parser = JSON::ResumableParser.new i = 0 while i < token.bytesize parser << token.byteslice(i, chunk_size) i += chunk_size parser.parse end end CHUNK = 128 sizes = [100, 16_000, 32_000, 64_000, 128_000] Benchmark.ips do |x| x.config(time: 3, warmup: 1) sizes.each do |n| number = "1" * n string = '"' + ("a" * n) x.report("number #{n} digits") { feed_incomplete(number, CHUNK) } x.report("string #{n} chars") { feed_incomplete(string, CHUNK) } end end ``` Per-iteration time, before vs. after this change: | Input (128-byte chunks) | Before | After | Change | |-------------------------|---------:|---------:|-----------------| | number, 100 digits | 949 ns | 414 ns | negligible | | number, 16,000 digits | 16.4 ms | 0.138 ms | ~119x faster | | number, 32,000 digits | 94.8 ms | 0.531 ms | ~178x faster | | number, 64,000 digits | 540.8 ms | 2.05 ms | ~264x faster | | number, 128,000 digits | 3.07 s | 8.13 ms | ~378x faster | | string, 100 chars | 411 ns | 391 ns | ~1x (unchanged) | | string, 16,000 chars | 49.4 μs | 47.7 μs | ~1x (unchanged) | | string, 32,000 chars | 169.7 μs | 165.8 μs | ~1x (unchanged) | | string, 64,000 chars | 621 μs | 616 μs | ~1x (unchanged) | | string, 128,000 chars | 2.38 ms | 2.35 ms | ~1x (unchanged) | The string rows are unchanged, confirming the change is number-specific. ## Fix Defer the decode. `json_parse_number` still scans the digits (cheap, and enough to advance the cursor so the caller can detect an incomplete number), but when the scan reaches the end of the buffer in resumable mode it returns `Qundef` without decoding. The caller already rewinds and returns `false` in exactly that case, so the decoded value was never used there. The number is decoded once, when a following byte proves it complete. Non-resumable `JSON.parse` is unaffected: it passes `resumable = false`, so the early return is never taken and the number is decoded as before. A test checks that large integers and floats split across feeds still decode to the correct value. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01PpeQSEFkF1X1Uuzjx9BsYe https://github.com/ruby/json/commit/d1299d42ac --- ext/json/parser/parser.c | 16 +++++++++++++--- test/json/resumable_parser_test.rb | 17 +++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ext/json/parser/parser.c b/ext/json/parser/parser.c index 9b7b4be3927ed5..d1c90ff8d77169 100644 --- a/ext/json/parser/parser.c +++ b/ext/json/parser/parser.c @@ -1457,7 +1457,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat return (int)(state->cursor - start); } -static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start) +static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable) { bool integer = true; const char first_digit = *state->cursor; @@ -1514,6 +1514,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } } + // A number touching the end of the buffer may still grow in a later chunk, + // so the caller will rewind and wait. Decoding it now would build a value + // -- for a long run of digits, an expensive bignum -- only to discard it, + // and repeating that on every resumed chunk is quadratic in the number's + // length. The digit scan above already advanced the cursor, which is all + // the caller needs to detect the incomplete number. + if (RB_UNLIKELY(resumable && eos(state))) { + return Qundef; + } + if (integer) { return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor); } @@ -1638,7 +1648,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '-': { state->cursor++; - value = json_parse_number(state, config, true, value_start); + value = json_parse_number(state, config, true, value_start, resumable); if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) { state->cursor = value_start; @@ -1661,7 +1671,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - value = json_parse_number(state, config, false, value_start); + value = json_parse_number(state, config, false, value_start, resumable); // Top level numbers are ambiguous when parsing streams, we can't // know if we parsed all the digits if we hit EOS. diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 5d7bf38bbf8ccc..b3c82c167c50a7 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -175,6 +175,23 @@ def test_parse_byte_by_byte_numbers assert_resumed_parsing('123 ', trailing_bytes: 1) end + def test_large_numbers_split_across_feeds_are_decoded_correctly + { + '12345678901234567890123456789012345678901234567890 ' => 12345678901234567890123456789012345678901234567890, + '-98765432109876543210987654321 ' => -98765432109876543210987654321, + '3.14159265358979323846264338327950288 ' => 3.14159265358979323846264338327950288, + '-1.5e-300 ' => -1.5e-300, + }.each do |doc, expected| + parser = new_parser + value = nil + doc.each_char do |char| + parser << char + value = parser.value if parser.parse + end + assert_equal expected, value, doc.inspect + end + end + def test_nul_byte_is_a_syntax_error # A NUL byte in a structural position must raise, not stall forever waiting for more input # (peek() returns 0 both at EOS and for a literal NUL byte). From edf0ea0355e3334b9f07f0a3af91bf01c70046c8 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 9 Jul 2026 20:40:09 +0900 Subject: [PATCH 7/8] Fix indents [ci skip] --- pack.c | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/pack.c b/pack.c index 378c5e17a2315f..d89075803e01c1 100644 --- a/pack.c +++ b/pack.c @@ -213,34 +213,34 @@ skip_to_eol(const char *p, const char *pend) static char * pack_modifiers(const char *p, char type, int *natint, int *explicit_endian) { - while (1) { - switch (*p) { - case '_': - case '!': - if (strchr(natstr, type)) { + while (1) { + switch (*p) { + case '_': + case '!': + if (strchr(natstr, type)) { #ifdef NATINT_PACK - *natint = 1; + *natint = 1; #endif - p++; - } - else { - rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr); - } - break; - - case '<': - case '>': - if (!strchr(endstr, type)) { - rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr); - } - if (*explicit_endian) { - rb_raise(rb_eRangeError, "Can't use both '<' and '>'"); - } - *explicit_endian = *p++; - break; - default: - return (char *)p; - } + p++; + } + else { + rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr); + } + break; + + case '<': + case '>': + if (!strchr(endstr, type)) { + rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr); + } + if (*explicit_endian) { + rb_raise(rb_eRangeError, "Can't use both '<' and '>'"); + } + *explicit_endian = *p++; + break; + default: + return (char *)p; + } } } From 835ad5c09e3647cf1ab349f199d98affdbbabace Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Thu, 9 Jul 2026 20:43:56 +0900 Subject: [PATCH 8/8] Fix a potential out-of-bounds read --- pack.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pack.c b/pack.c index d89075803e01c1..56af4ea099cae6 100644 --- a/pack.c +++ b/pack.c @@ -208,12 +208,12 @@ skip_to_eol(const char *p, const char *pend) (ISSPACE(type) || (type == '#' && (p = skip_to_eol(p, pend), 1))) #ifndef NATINT_PACK -# define pack_modifiers(p, t, n, e) pack_modifiers(p, t, e) +# define pack_modifiers(p, pe, t, n, e) pack_modifiers(p, pe, t, e) #endif -static char * -pack_modifiers(const char *p, char type, int *natint, int *explicit_endian) +static const char * +pack_modifiers(const char *p, const char *pend, char type, int *natint, int *explicit_endian) { - while (1) { + while (p < pend) { switch (*p) { case '_': case '!': @@ -242,6 +242,7 @@ pack_modifiers(const char *p, char type, int *natint, int *explicit_endian) return (char *)p; } } + return p; } static VALUE @@ -289,7 +290,7 @@ pack_pack(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) #endif if (skip_blank(p, type)) continue; - p = pack_modifiers(p, type, &natint, &explicit_endian); + p = pack_modifiers(p, pend, type, &natint, &explicit_endian); if (*p == '*') { /* set data length */ len = strchr("@Xxu", type) ? 0 @@ -1046,7 +1047,7 @@ pack_unpack_internal(VALUE str, VALUE fmt, VALUE ofs, enum unpack_mode mode) int star = 0; if (skip_blank(p, type)) continue; - p = pack_modifiers(p, type, &natint, &explicit_endian); + p = pack_modifiers(p, pend, type, &natint, &explicit_endian); if (p >= pend) len = 1;