diff --git a/NEWS.md b/NEWS.md index c666338ffafbe0..0d725703b9c9f9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -43,6 +43,10 @@ Note: We're only listing outstanding class updates. * `MatchData#integer_at` is added. It converts the matched substring to integer and return the result. [[Feature #21932]] +* ObjectSpace + + * `ObjectSpace._id2ref` was removed. [[Feature #22135]] + * Regexp * All instances of `Regexp` are now frozen, not just literals. diff --git a/bootstraptest/test_objectspace.rb b/bootstraptest/test_objectspace.rb index 63a8d9932287f6..bde5e9c0b409e7 100644 --- a/bootstraptest/test_objectspace.rb +++ b/bootstraptest/test_objectspace.rb @@ -1,18 +1,3 @@ -assert_normal_exit %q{ - eval("", TOPLEVEL_BINDING) - minobj = ObjectSpace.to_enum(:each_object).min_by {|a| a.object_id } - maxobj = ObjectSpace.to_enum(:each_object).max_by {|a| a.object_id } - (((minobj.object_id-100)..(minobj.object_id+100))+ - ((maxobj.object_id-100)..(maxobj.object_id+100))).each {|id| - begin - o = ObjectSpace._id2ref(id) - rescue RangeError - next - end - o.inspect if defined?(o.inspect) - } -}, '[ruby-dev:31911]' - assert_normal_exit %q{ ary = (1..10).to_a ary.permutation(2) {|x| @@ -44,12 +29,3 @@ Thread.new {} end }, '[ruby-core:37858]' - -assert_equal 'ok', %q{ - objects_and_ids = 1000.times.map { o = Object.new; [o, o.object_id] } - objects_and_ids.each { |expected, id| - actual = ObjectSpace._id2ref(id) - raise "expected #{expected.inspect}, got #{actual.inspect}" unless actual.equal?(expected) - } - 'ok' -} diff --git a/bootstraptest/test_ractor.rb b/bootstraptest/test_ractor.rb index 4fe90703fc4c54..22d2ee47deb1ce 100644 --- a/bootstraptest/test_ractor.rb +++ b/bootstraptest/test_ractor.rb @@ -1256,37 +1256,6 @@ class C }.value } -# ObjectSpace._id2ref can not handle unshareable objects with Ractors -assert_equal 'ok', <<~'RUBY', frozen_string_literal: false - s = 'hello' - - Ractor.new s.object_id do |id ;s| - begin - s = ObjectSpace._id2ref(id) - rescue => e - :ok - end - end.value -RUBY - -# Inserting into the id2ref table should be Ractor-safe -assert_equal 'ok', <<~'RUBY' - # Force all calls to Kernel#object_id to insert into the id2ref table - obj = Object.new - ObjectSpace._id2ref(obj.object_id) rescue nil - - 10.times.map do - Ractor.new do - 10_000.times do - a = Object.new - a.object_id - end - end - end.map(&:value) - - :ok -RUBY - # Ractor.make_shareable(obj) assert_equal 'true', <<~'RUBY', frozen_string_literal: false class C @@ -2175,19 +2144,6 @@ def ==(o) roundtripped_obj.instance_variable_get(:@array1) == [1] ? :ok : roundtripped_obj } -# move object with generic ivars and existing id2ref table -# [Bug #21664] -assert_equal 'ok', %q{ - obj = [1] - obj.instance_variable_set("@field", :ok) - ObjectSpace._id2ref(obj.object_id) # build id2ref table - - ractor = Ractor.new { Ractor.receive } - ractor.send(obj, move: true) - obj = ractor.value - obj.instance_variable_get("@field") -} - # copy object with complex generic ivars assert_equal 'ok', %q{ # Make Array complex diff --git a/ext/json/parser/parser.c b/ext/json/parser/parser.c index 9b7b4be3927ed5..d1c90ff8d77169 100644 --- a/ext/json/parser/parser.c +++ b/ext/json/parser/parser.c @@ -1457,7 +1457,7 @@ static inline int json_parse_digits(JSON_ParserState *state, uint64_t *accumulat return (int)(state->cursor - start); } -static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start) +static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig *config, bool negative, const char *start, bool resumable) { bool integer = true; const char first_digit = *state->cursor; @@ -1514,6 +1514,16 @@ static inline VALUE json_parse_number(JSON_ParserState *state, JSON_ParserConfig } } + // A number touching the end of the buffer may still grow in a later chunk, + // so the caller will rewind and wait. Decoding it now would build a value + // -- for a long run of digits, an expensive bignum -- only to discard it, + // and repeating that on every resumed chunk is quadratic in the number's + // length. The digit scan above already advanced the cursor, which is all + // the caller needs to detect the incomplete number. + if (RB_UNLIKELY(resumable && eos(state))) { + return Qundef; + } + if (integer) { return json_decode_integer(mantissa, mantissa_digits, negative, start, state->cursor); } @@ -1638,7 +1648,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo case '-': { state->cursor++; - value = json_parse_number(state, config, true, value_start); + value = json_parse_number(state, config, true, value_start, resumable); if (RB_UNLIKELY(UNDEF_P(value) && config->allow_nan && peek(state) == 'I')) { state->cursor = value_start; @@ -1661,7 +1671,7 @@ ALWAYS_INLINE(static) bool json_parse_any(JSON_ParserState *state, JSON_ParserCo } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { - value = json_parse_number(state, config, false, value_start); + value = json_parse_number(state, config, false, value_start, resumable); // Top level numbers are ambiguous when parsing streams, we can't // know if we parsed all the digits if we hit EOS. diff --git a/gc.c b/gc.c index d12e37f4346bed..e521369caa72ef 100644 --- a/gc.c +++ b/gc.c @@ -586,7 +586,7 @@ rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val) #endif static const char *obj_type_name(VALUE obj); -static st_table *id2ref_tbl; + #include "gc/default/default.c" #if USE_MODULAR_GC && !defined(HAVE_DLOPEN) @@ -1353,7 +1353,7 @@ rb_gc_imemo_needs_cleanup_p(VALUE obj) return ((rb_imemo_tmpbuf_t *)obj)->ptr != NULL; case imemo_fields: - return FL_TEST_RAW(obj, OBJ_FIELD_HEAP) || (id2ref_tbl && rb_obj_shape_has_id(obj)); + return FL_TEST_RAW(obj, OBJ_FIELD_HEAP); } UNREACHABLE_RETURN(true); } @@ -1365,7 +1365,6 @@ rb_gc_imemo_needs_cleanup_p(VALUE obj) * Objects that return false are: * - Simple embedded objects without external allocations * - Objects without finalizers - * - Objects without object IDs registered in id2ref * - Objects without generic instance variables * * This is used by the GC sweep fast path to avoid function call overhead @@ -1405,7 +1404,6 @@ rb_gc_obj_needs_cleanup_p(VALUE obj) } shape_id_t shape_id = RBASIC_SHAPE_ID(obj); - if (id2ref_tbl && rb_shape_has_object_id(shape_id)) return true; switch (flags & RUBY_T_MASK) { case T_OBJECT: @@ -2054,7 +2052,6 @@ rb_objspace_garbage_object_p(VALUE obj) #define OBJ_ID_INCREMENT (RUBY_IMMEDIATE_MASK + 1) #define LAST_OBJECT_ID() (object_id_counter * OBJ_ID_INCREMENT) -static VALUE id2ref_value = 0; #if SIZEOF_SIZE_T == SIZEOF_LONG_LONG static size_t object_id_counter = 1; @@ -2076,76 +2073,8 @@ generate_next_object_id(void) #endif } -void -rb_gc_obj_id_moved(VALUE obj) -{ - if (UNLIKELY(id2ref_tbl)) { - st_insert(id2ref_tbl, (st_data_t)rb_obj_id(obj), (st_data_t)obj); - } -} - -static int -object_id_cmp(st_data_t x, st_data_t y) -{ - if (RB_TYPE_P(x, T_BIGNUM)) { - return !rb_big_eql(x, y); - } - else { - return x != y; - } -} - -static st_index_t -object_id_hash(st_data_t n) -{ - return FIX2LONG(rb_hash((VALUE)n)); -} - -static const struct st_hash_type object_id_hash_type = { - object_id_cmp, - object_id_hash, -}; - static void gc_mark_tbl_no_pin(st_table *table); -static void -id2ref_tbl_mark(void *data) -{ - st_table *table = (st_table *)data; - if (UNLIKELY(!RB_POSFIXABLE(LAST_OBJECT_ID()))) { - // It's very unlikely, but if enough object ids were generated, keys may be T_BIGNUM - rb_mark_set(table); - } - // We purposely don't mark values, as they are weak references. - // rb_gc_obj_free_vm_weak_references takes care of cleaning them up. -} - -static size_t -id2ref_tbl_memsize(const void *data) -{ - return rb_st_memsize(data); -} - -static void -id2ref_tbl_free(void *data) -{ - id2ref_tbl = NULL; // clear global ref - st_table *table = (st_table *)data; - st_free_table(table); -} - -static const rb_data_type_t id2ref_tbl_type = { - .wrap_struct_name = "VM/_id2ref_table", - .function = { - .dmark = id2ref_tbl_mark, - .dfree = id2ref_tbl_free, - .dsize = id2ref_tbl_memsize, - // dcompact function not required because the table is reference updated - // in rb_gc_vm_weak_table_foreach - }, - .flags = RUBY_TYPED_WB_PROTECTED | RUBY_TYPED_FREE_IMMEDIATELY -}; - static VALUE class_object_id(VALUE klass) { @@ -2157,9 +2086,6 @@ class_object_id(VALUE klass) if (existing_id) { id = existing_id; } - else if (RB_UNLIKELY(id2ref_tbl)) { - st_insert(id2ref_tbl, id, klass); - } RB_GC_VM_UNLOCK(lock_lev); } return id; @@ -2203,11 +2129,6 @@ object_id0(VALUE obj) RUBY_ASSERT(rb_obj_shape_has_id(obj)); - if (RB_UNLIKELY(id2ref_tbl)) { - RB_VM_LOCKING() { - st_insert(id2ref_tbl, (st_data_t)id, (st_data_t)obj); - } - } return id; } @@ -2238,123 +2159,9 @@ object_id(VALUE obj) return object_id0(obj); } -static void -build_id2ref_i(VALUE obj, void *data) -{ - st_table *id2ref_tbl = (st_table *)data; - - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (RCLASS(obj)->object_id) { - st_insert(id2ref_tbl, RCLASS(obj)->object_id, obj); - } - break; - case T_IMEMO: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (IMEMO_TYPE_P(obj, imemo_fields) && rb_obj_shape_has_id(obj)) { - st_insert(id2ref_tbl, rb_obj_id(obj), rb_imemo_fields_owner(obj)); - } - break; - case T_OBJECT: - RUBY_ASSERT(!rb_objspace_garbage_object_p(obj)); - if (rb_obj_shape_has_id(obj)) { - st_insert(id2ref_tbl, rb_obj_id(obj), obj); - } - break; - default: - // For generic_fields, the T_IMEMO/fields is responsible for populating the entry. - break; - } -} - -static VALUE -object_id_to_ref(void *objspace_ptr, VALUE object_id) -{ - rb_objspace_t *objspace = objspace_ptr; - - unsigned int lev = RB_GC_VM_LOCK(); - - if (!id2ref_tbl) { - rb_gc_vm_barrier(); // stop other ractors - - // GC Must not trigger while we build the table, otherwise if we end - // up freeing an object that had an ID, we might try to delete it from - // the table even though it wasn't inserted yet. - st_table *tmp_id2ref_tbl = st_init_table(&object_id_hash_type); - VALUE tmp_id2ref_value = TypedData_Wrap_Struct(0, &id2ref_tbl_type, tmp_id2ref_tbl); - - // build_id2ref_i will most certainly malloc, which could trigger GC and sweep - // objects we just added to the table. - // By calling rb_gc_disable() we also save having to handle potentially garbage objects. - bool gc_disabled = RTEST(rb_gc_disable()); - { - id2ref_tbl = tmp_id2ref_tbl; - id2ref_value = tmp_id2ref_value; - - rb_gc_impl_each_object(objspace, build_id2ref_i, (void *)id2ref_tbl); - } - if (!gc_disabled) rb_gc_enable(); - } - - VALUE obj; - bool found = st_lookup(id2ref_tbl, object_id, &obj) && !rb_gc_impl_garbage_object_p(objspace, obj); - - RB_GC_VM_UNLOCK(lev); - - if (found) { - return obj; - } - - if (rb_funcall(object_id, rb_intern(">="), 1, ULL2NUM(LAST_OBJECT_ID()))) { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } - else { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is a recycled object", rb_funcall(object_id, rb_intern("to_s"), 1, INT2FIX(10))); - } -} - static inline void obj_free_object_id(VALUE obj) { - VALUE obj_id = 0; - if (RB_UNLIKELY(id2ref_tbl)) { - switch (BUILTIN_TYPE(obj)) { - case T_CLASS: - case T_MODULE: - obj_id = RCLASS(obj)->object_id; - break; - case T_IMEMO: - if (!IMEMO_TYPE_P(obj, imemo_fields)) { - return; - } - // fallthrough - case T_OBJECT: - { - shape_id_t shape_id = RBASIC_SHAPE_ID(obj); - if (rb_shape_has_object_id(shape_id)) { - obj_id = object_id_get(obj, shape_id); - } - break; - } - default: - // For generic_fields, the T_IMEMO/fields is responsible for freeing the id. - return; - } - - if (RB_UNLIKELY(obj_id)) { - RUBY_ASSERT(FIXNUM_P(obj_id) || RB_TYPE_P(obj_id, T_BIGNUM)); - - if (!st_delete(id2ref_tbl, (st_data_t *)&obj_id, NULL)) { - // The the object is a T_IMEMO/fields, then it's possible the actual object - // has been garbage collected already. - if (!RB_TYPE_P(obj, T_IMEMO)) { - rb_bug("Object ID seen, but not in _id2ref table: object_id=%llu object=%s", NUM2ULL(obj_id), rb_obj_info(obj)); - } - } - } - } } void @@ -2393,67 +2200,6 @@ rb_gc_obj_free_vm_weak_references(VALUE obj) } } -/* - * call-seq: - * ObjectSpace._id2ref(object_id) -> an_object - * - * Converts an object id to a reference to the object. May not be - * called on an object id passed as a parameter to a finalizer. - * - * s = "I am a string" #=> "I am a string" - * r = ObjectSpace._id2ref(s.object_id) #=> "I am a string" - * r == s #=> true - * - * On multi-ractor mode, if the object is not shareable, it raises - * RangeError. - * - * This method is deprecated and should no longer be used. - */ - -static VALUE -id2ref(VALUE objid) -{ - objid = rb_to_int(objid); - if (FIXNUM_P(objid) || rb_big_size(objid) <= SIZEOF_VOIDP) { - VALUE ptr = (VALUE)NUM2PTR(objid); - if (SPECIAL_CONST_P(ptr)) { - if (ptr == Qtrue) return Qtrue; - if (ptr == Qfalse) return Qfalse; - if (NIL_P(ptr)) return Qnil; - if (FIXNUM_P(ptr)) return ptr; - if (FLONUM_P(ptr)) return ptr; - - if (SYMBOL_P(ptr)) { - // Check that the symbol is valid - if (rb_static_id_valid_p(SYM2ID(ptr))) { - return ptr; - } - else { - rb_raise(rb_eRangeError, "%p is not a symbol id value", (void *)ptr); - } - } - - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is not an id value", rb_int2str(objid, 10)); - } - } - - VALUE obj = object_id_to_ref(rb_gc_get_objspace(), objid); - if (!rb_multi_ractor_p() || rb_ractor_shareable_p(obj)) { - return obj; - } - else { - rb_raise(rb_eRangeError, "%+"PRIsVALUE" is the id of an unshareable object on multi-ractor", rb_int2str(objid, 10)); - } -} - -/* :nodoc: */ -static VALUE -os_id2ref(VALUE os, VALUE objid) -{ - rb_category_warn(RB_WARN_CATEGORY_DEPRECATED, "ObjectSpace._id2ref is deprecated"); - return id2ref(objid); -} - static VALUE rb_find_object_id(void *objspace, VALUE obj, VALUE (*get_heap_object_id)(VALUE)) { @@ -4208,33 +3954,6 @@ vm_weak_table_sym_set_foreach(VALUE *sym_ptr, void *data) struct st_table *rb_generic_fields_tbl_get(void); -static int -vm_weak_table_id2ref_foreach(st_data_t key, st_data_t value, st_data_t data, int error) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - if (!iter_data->weak_only && !FIXNUM_P((VALUE)key)) { - int ret = iter_data->callback((VALUE)key, iter_data->data); - if (ret != ST_CONTINUE) return ret; - } - - return iter_data->callback((VALUE)value, iter_data->data); -} - -static int -vm_weak_table_id2ref_foreach_update(st_data_t *key, st_data_t *value, st_data_t data, int existing) -{ - struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data; - - iter_data->update_callback((VALUE *)value, iter_data->data); - - if (!iter_data->weak_only && !FIXNUM_P((VALUE)*key)) { - iter_data->update_callback((VALUE *)key, iter_data->data); - } - - return ST_CONTINUE; -} - static int vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data) { @@ -4356,17 +4075,6 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback, ); break; } - case RB_GC_VM_ID2REF_TABLE: { - if (id2ref_tbl) { - st_foreach_with_replace( - id2ref_tbl, - vm_weak_table_id2ref_foreach, - vm_weak_table_id2ref_foreach_update, - (st_data_t)&foreach_data - ); - } - break; - } case RB_GC_VM_GENERIC_FIELDS_TABLE: { st_table *generic_fields_tbl = rb_generic_fields_tbl_get(); if (generic_fields_tbl) { @@ -5870,8 +5578,6 @@ Init_GC(void) #endif #undef rb_intern - rb_gc_register_address(&id2ref_value); - malloc_offset = gc_compute_malloc_offset(); rb_mGC = rb_define_module("GC"); @@ -5883,8 +5589,6 @@ Init_GC(void) rb_define_module_function(rb_mObjSpace, "define_finalizer", define_final, -1); rb_define_module_function(rb_mObjSpace, "undefine_finalizer", undefine_final, 1); - rb_define_module_function(rb_mObjSpace, "_id2ref", os_id2ref, 1); - rb_vm_register_special_exception(ruby_error_nomemory, rb_eNoMemError, "failed to allocate memory"); rb_define_method(rb_cBasicObject, "__id__", rb_obj_id, 0); diff --git a/gc/gc.h b/gc/gc.h index d8c351c7e92754..3de9b3f165ddb3 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -48,7 +48,6 @@ enum rb_gc_vm_weak_tables { RB_GC_VM_CI_TABLE, RB_GC_VM_OVERLOADED_CME_TABLE, RB_GC_VM_GLOBAL_SYMBOLS_TABLE, - RB_GC_VM_ID2REF_TABLE, RB_GC_VM_GENERIC_FIELDS_TABLE, RB_GC_VM_FROZEN_STRINGS_TABLE, RB_GC_VM_WEAK_TABLE_COUNT diff --git a/include/ruby/internal/core/rarray.h b/include/ruby/internal/core/rarray.h index b47b63d51b05e2..d70535e9d9ba41 100644 --- a/include/ruby/internal/core/rarray.h +++ b/include/ruby/internal/core/rarray.h @@ -45,7 +45,6 @@ /** @cond INTERNAL_MACRO */ #define RARRAY_EMBED_FLAG RARRAY_EMBED_FLAG #define RARRAY_EMBED_LEN_MASK RARRAY_EMBED_LEN_MASK -#define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX #define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT /** @endcond */ #define RARRAY_LEN rb_array_len /**< @alias{rb_array_len} */ diff --git a/internal/gc.h b/internal/gc.h index 8a3dbd1884b4bf..9a137d2609554f 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -199,7 +199,6 @@ RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add(size_t, size_t, size_t); void *rb_xrealloc_mul_add(const void *, size_t, size_t, size_t); RUBY_ATTR_MALLOC void *rb_xmalloc_mul_add_mul(size_t, size_t, size_t, size_t); RUBY_ATTR_MALLOC void *rb_xcalloc_mul_add_mul(size_t, size_t, size_t, size_t); -void rb_gc_obj_id_moved(VALUE obj); void rb_gc_register_pinning_obj(VALUE obj); rb_execution_context_t *rb_gc_get_ec(void); diff --git a/pack.c b/pack.c index 378c5e17a2315f..56af4ea099cae6 100644 --- a/pack.c +++ b/pack.c @@ -208,40 +208,41 @@ skip_to_eol(const char *p, const char *pend) (ISSPACE(type) || (type == '#' && (p = skip_to_eol(p, pend), 1))) #ifndef NATINT_PACK -# define pack_modifiers(p, t, n, e) pack_modifiers(p, t, e) +# define pack_modifiers(p, pe, t, n, e) pack_modifiers(p, pe, t, e) #endif -static char * -pack_modifiers(const char *p, char type, int *natint, int *explicit_endian) +static const char * +pack_modifiers(const char *p, const char *pend, char type, int *natint, int *explicit_endian) { - while (1) { - switch (*p) { - case '_': - case '!': - if (strchr(natstr, type)) { + while (p < pend) { + switch (*p) { + case '_': + case '!': + if (strchr(natstr, type)) { #ifdef NATINT_PACK - *natint = 1; + *natint = 1; #endif - p++; - } - else { - rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr); - } - break; - - case '<': - case '>': - if (!strchr(endstr, type)) { - rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr); - } - if (*explicit_endian) { - rb_raise(rb_eRangeError, "Can't use both '<' and '>'"); - } - *explicit_endian = *p++; - break; - default: - return (char *)p; - } + p++; + } + else { + rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, natstr); + } + break; + + case '<': + case '>': + if (!strchr(endstr, type)) { + rb_raise(rb_eArgError, "'%c' allowed only after types %s", *p, endstr); + } + if (*explicit_endian) { + rb_raise(rb_eRangeError, "Can't use both '<' and '>'"); + } + *explicit_endian = *p++; + break; + default: + return (char *)p; + } } + return p; } static VALUE @@ -289,7 +290,7 @@ pack_pack(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer) #endif if (skip_blank(p, type)) continue; - p = pack_modifiers(p, type, &natint, &explicit_endian); + p = pack_modifiers(p, pend, type, &natint, &explicit_endian); if (*p == '*') { /* set data length */ len = strchr("@Xxu", type) ? 0 @@ -1046,7 +1047,7 @@ pack_unpack_internal(VALUE str, VALUE fmt, VALUE ofs, enum unpack_mode mode) int star = 0; if (skip_blank(p, type)) continue; - p = pack_modifiers(p, type, &natint, &explicit_endian); + p = pack_modifiers(p, pend, type, &natint, &explicit_endian); if (p >= pend) len = 1; diff --git a/pathname_builtin.rb b/pathname_builtin.rb index 9b31b73b87af43..ab3132725db541 100644 --- a/pathname_builtin.rb +++ b/pathname_builtin.rb @@ -581,17 +581,25 @@ def mountpoint? end end - # The opposite of Pathname#absolute? + # :markup: markdown + # + # call-seq: + # relative? -> true or false # - # It returns +false+ if the pathname begins with a slash. + # Returns whether `self` contains a relative path: + # + # ```ruby + # Pathname('lib').relative? # => true + # Pathname('/home').relative? # => false + # ``` # - # p = Pathname.new('/im/sure') - # p.relative? - # #=> false + # The result is OS-dependent for some paths: + # + # ```ruby + # Pathname('C:/').relative? # => false # On Windows. + # Pathname('C:/').relative? # => true # Elsewhere. + # ``` # - # p = Pathname.new('not/so/sure') - # p.relative? - # #=> true def relative? !absolute? end @@ -895,19 +903,41 @@ def each_child(with_directory=true, &b) children(with_directory).each(&b) end + # :markup: markdown + # + # call-seq: + # relative_path_from(source) -> new_pathname # - # Returns a relative path from the given +base_directory+ to the receiver. + # Returns a pathname containing the relative filesystem path from the given `source` + # to the path in `self`; + # `source` must be a directory path or a pathname containing a directory path: # - # If +self+ is absolute, then +base_directory+ must be absolute too. + # ```ruby + # Pathname('.').relative_path_from('doc/language') # => # + # Pathname('doc/language').relative_path_from('.') # => # + # Pathname('doc').relative_path_from('doc/language') # => # + # ``` # - # If +self+ is relative, then +base_directory+ must be relative too. + # The paths need not exist: # - # This method doesn't access the filesystem. It assumes no symlinks. + # ```ruby + # Pathname('nosuch').relative_path_from('nosuch/foo/bar/baz') + # # => # + # ``` # - # ArgumentError is raised when it cannot find a relative path. + # The two paths must be either both absolute or both relative: # - # Note that this method does not handle situations where the case sensitivity - # of the filesystem in use differs from the operating system default. + # ```ruby + # Pathname('/var').relative_path_from('/etc') # => # + # Pathname('/var').relative_path_from('doc') # Raises ArgumentError + # Pathname('doc').relative_path_from('/etc') # Raises ArgumentError + # ``` + # + # Raises an exception if there is no such relative path: + # + # ```ruby + # Pathname('foo').relative_path_from('..') # Raises ArgumentError + # ``` # def relative_path_from(base_directory) base_directory = Pathname.new(base_directory) unless base_directory.is_a? Pathname diff --git a/ractor.c b/ractor.c index fc6694c45c7f84..22b6432c4b1e81 100644 --- a/ractor.c +++ b/ractor.c @@ -2059,8 +2059,6 @@ move_leave(VALUE obj, struct obj_traverse_replace_data *data) rb_replace_generic_ivar(data->replacement, obj); } - rb_gc_obj_id_moved(data->replacement); - VALUE flags = T_OBJECT | FL_FREEZE | (RBASIC(obj)->flags & FL_PROMOTED); // Avoid mutations using bind_call, etc. diff --git a/spec/ruby/core/objectspace/_id2ref_spec.rb b/spec/ruby/core/objectspace/_id2ref_spec.rb index a9fd526b7d0f68..19461b8729f359 100644 --- a/spec/ruby/core/objectspace/_id2ref_spec.rb +++ b/spec/ruby/core/objectspace/_id2ref_spec.rb @@ -1,6 +1,6 @@ require_relative '../../spec_helper' -ruby_version_is "4.0" do +ruby_version_is "4.0"..."4.1" do describe "ObjectSpace._id2ref" do it "is deprecated" do id = nil.object_id diff --git a/test/json/resumable_parser_test.rb b/test/json/resumable_parser_test.rb index 5d7bf38bbf8ccc..b3c82c167c50a7 100644 --- a/test/json/resumable_parser_test.rb +++ b/test/json/resumable_parser_test.rb @@ -175,6 +175,23 @@ def test_parse_byte_by_byte_numbers assert_resumed_parsing('123 ', trailing_bytes: 1) end + def test_large_numbers_split_across_feeds_are_decoded_correctly + { + '12345678901234567890123456789012345678901234567890 ' => 12345678901234567890123456789012345678901234567890, + '-98765432109876543210987654321 ' => -98765432109876543210987654321, + '3.14159265358979323846264338327950288 ' => 3.14159265358979323846264338327950288, + '-1.5e-300 ' => -1.5e-300, + }.each do |doc, expected| + parser = new_parser + value = nil + doc.each_char do |char| + parser << char + value = parser.value if parser.parse + end + assert_equal expected, value, doc.inspect + end + end + def test_nul_byte_is_a_syntax_error # A NUL byte in a structural position must raise, not stall forever waiting for more input # (peek() returns 0 both at EOS and for a literal NUL byte). diff --git a/test/ruby/test_objectspace.rb b/test/ruby/test_objectspace.rb index a479547599a03a..033ad0f0de56c0 100644 --- a/test/ruby/test_objectspace.rb +++ b/test/ruby/test_objectspace.rb @@ -2,77 +2,6 @@ require 'test/unit' class TestObjectSpace < Test::Unit::TestCase - def self.deftest_id2ref(obj) - /:(\d+)/ =~ caller[0] - file = $` - line = $1.to_i - code = <<"End" - define_method("test_id2ref_#{line}") {\ - o = EnvUtil.suppress_warning { ObjectSpace._id2ref(obj.object_id) } - assert_same(obj, o, "didn't round trip: \#{obj.inspect}");\ - } -End - eval code, binding, file, line - end - - deftest_id2ref(-0x4000000000000001) - deftest_id2ref(-0x4000000000000000) - deftest_id2ref(-0x40000001) - deftest_id2ref(-0x40000000) - deftest_id2ref(-1) - deftest_id2ref(0) - deftest_id2ref(1) - deftest_id2ref(0x3fffffff) - deftest_id2ref(0x40000000) - deftest_id2ref(0x3fffffffffffffff) - deftest_id2ref(0x4000000000000000) - deftest_id2ref(:a) - deftest_id2ref(:abcdefghijilkjl) - deftest_id2ref(:==) - deftest_id2ref(Object.new) - deftest_id2ref(self) - deftest_id2ref(true) - deftest_id2ref(false) - deftest_id2ref(nil) - - def test_id2ref_liveness - assert_normal_exit <<-EOS - ids = [] - 10.times{ - 1_000.times{ - ids << 'hello'.object_id - } - objs = ids.map{|id| - begin - ObjectSpace._id2ref(id) - rescue RangeError - nil - end - } - GC.start - objs.each{|e| e.inspect} - } - EOS - end - - def test_id2ref_invalid_argument - msg = /no implicit conversion/ - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(nil) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(false) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(true) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(:a) } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref("0") } } - assert_raise_with_message(TypeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(Object.new) } } - end - - def test_id2ref_invalid_symbol_id - # RB_STATIC_SYM_P checks for static symbols by checking that the bottom - # 8 bits of the object is equal to RUBY_SYMBOL_FLAG, so we need to make - # sure that the bottom 8 bits remain unchanged. - msg = /is not a symbol id value/ - assert_raise_with_message(RangeError, msg) { EnvUtil.suppress_warning { ObjectSpace._id2ref(:a.object_id + 256) } } - end - def test_count_objects h = {} ObjectSpace.count_objects(h) @@ -284,21 +213,6 @@ def test_each_object_recursive_key end; end - def test_id2ref_table_build - assert_separately([], <<-End) - 10.times do - Object.new.object_id - end - - GC.start(immediate_mark: false) - - obj = Object.new - EnvUtil.suppress_warning do - assert_equal obj, ObjectSpace._id2ref(obj.object_id) - end - End - end - def test_each_object_singleton_class assert_separately([], <<-End) class C diff --git a/vm.c b/vm.c index 85a22d40048261..86c0e3391553a6 100644 --- a/vm.c +++ b/vm.c @@ -2858,17 +2858,22 @@ rb_zjit_materialize_frames(const rb_execution_context_t *ec, rb_control_frame_t // jit_return, so this restores older ZJIT frames from stack maps. int32_t stack_size = (int32_t)jit_frame->stack_size; if (stack_size > 0) { - VALUE *stack = cfp->sp - stack_size; + VALUE *stack = cfp->sp; for (int32_t i = 0; i < stack_size; i++) { VALUE entry = jit_frame->stack[i]; if (ZJIT_STACK_MAP_VREG_P(entry)) { // Decode a native stack slot offset generated by ZJIT's backend. // It's an offset from NATIVE_BASE_PTR, which is copied into // cfp->jit_return, to the encoded stack slot. - stack[i] = ((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_VREG_INDEX(entry)]; + stack--; + *stack = ((VALUE *)cfp->jit_return)[-(ssize_t)ZJIT_STACK_MAP_VREG_INDEX(entry)]; + } + else if (ZJIT_STACK_MAP_SKIP_P(entry)) { + stack -= ZJIT_STACK_MAP_SKIP_SIZE(entry); } else { - stack[i] = entry; + stack--; + *stack = entry; } } } diff --git a/zjit.h b/zjit.h index 34ebdc095c6be9..586cf28a20c598 100644 --- a/zjit.h +++ b/zjit.h @@ -9,10 +9,12 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif -// Stack map entries are either immediate Ruby VALUEs or tagged native-stack -// locations. Stack maps never contain heap VALUEs, so 0x08 is available: it is -// not Qfalse (0), and its low 3 bits are zero, so RB_SPECIAL_CONST_P is false. +// Stack map entries are either immediate Ruby VALUEs, tagged native-stack +// locations, or tagged skip counts. Stack maps never contain heap VALUEs, so +// these tags are available: they are not Qfalse (0), and their low 3 bits are +// zero, so RB_SPECIAL_CONST_P is false. #define ZJIT_STACK_MAP_VREG_TAG 0x08 +#define ZJIT_STACK_MAP_SKIP_TAG 0x10 #define ZJIT_STACK_MAP_TAG_MASK 0xff #define ZJIT_STACK_MAP_SHIFT 8 @@ -28,6 +30,18 @@ ZJIT_STACK_MAP_VREG_INDEX(VALUE entry) return entry >> ZJIT_STACK_MAP_SHIFT; } +static inline bool +ZJIT_STACK_MAP_SKIP_P(VALUE entry) +{ + return (entry & ZJIT_STACK_MAP_TAG_MASK) == ZJIT_STACK_MAP_SKIP_TAG; +} + +static inline size_t +ZJIT_STACK_MAP_SKIP_SIZE(VALUE entry) +{ + return entry >> ZJIT_STACK_MAP_SHIFT; +} + // JITFrame is defined here as the single source of truth and imported into // Rust via bindgen. C code reads fields directly; Rust uses an impl block. typedef struct zjit_jit_frame { @@ -43,12 +57,11 @@ typedef struct zjit_jit_frame { // Always false for C frames. bool materialize_block_code; - // Number of Ruby stack slots described by stack[]. - // rb_zjit_materialize_frames() copies them to cfp->sp - stack_size. + // Number of stack map entries in stack[]. uint32_t stack_size; // Flexible array of stack map entries. Each entry is either an immediate - // VALUE or a tagged native-stack index from cfp->jit_return for a value - // kept by the JIT. + // VALUE, a tagged native-stack index from cfp->jit_return for a value + // kept by the JIT, or a tagged count of VM stack slots to skip. VALUE stack[]; } zjit_jit_frame_t; diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 801663a81f1c8e..95f7ee87cc5e9c 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -334,6 +334,7 @@ fn main() { .allowlist_type("zjit_struct_offsets") .allowlist_var("ZJIT_STACK_MAP_SHIFT") .allowlist_var("ZJIT_STACK_MAP_VREG_TAG") + .allowlist_var("ZJIT_STACK_MAP_SKIP_TAG") .allowlist_var("ZJIT_JIT_RETURN_C_FRAME") .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_complex_p") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 5c5c5cd3fa87f4..a91d0e8b375896 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1644,6 +1644,7 @@ impl Assembler { let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); asm.stack_state.num_spill_slots = num_stack_slots; + asm.stack_state.num_side_exit_stack_map_slots = asm.side_exit_stack_map_slots(&assignments); let stack_slot_count = asm.stack_state.stack_slot_count(); if stack_slot_count > Self::MAX_FRAME_STACK_SLOTS { return Err(CompileError::NativeStackTooLarge); @@ -1670,7 +1671,7 @@ impl Assembler { } // Update FrameSetup slot_count now that StackState knows the - // register allocator spill count. + // register allocator spill and side-exit capture counts. trace_compile_phase("count_stack_slots", || { for block in asm.basic_blocks.iter_mut() { for insn in block.insns.iter_mut() { @@ -1781,7 +1782,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } })); + let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], stack_map: None, recompile: None } })); asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index e5962b84dca775..f47fa06bdd5038 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -5,7 +5,7 @@ use std::mem::take; use std::rc::Rc; use crate::bitset::BitSet; use crate::codegen::{perf_symbol_range_start, perf_symbol_range_end, register_with_perf}; -use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, VALUE, ZJIT_STACK_MAP_SHIFT, ZJIT_STACK_MAP_VREG_TAG, vm_stack_canary, YarvInsnIdx, zjit_jit_frame, local_size_and_idx_to_ep_offset}; +use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, VALUE, ZJIT_STACK_MAP_SHIFT, ZJIT_STACK_MAP_SKIP_TAG, ZJIT_STACK_MAP_VREG_TAG, vm_stack_canary, YarvInsnIdx, zjit_jit_frame, local_size_and_idx_to_ep_offset}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -586,6 +586,10 @@ pub struct SideExit { pub stack: Vec, pub locals: Vec, pub iseq: IseqPtr, + /// Stack map for older inlined frames that are not written directly by this + /// side exit. The current frame's stack and locals are still handled by + /// `stack` and `locals` above. + pub stack_map: Option, /// If set, the side exit will profile the current instruction and invalidate /// the compiled ISEQ for recompilation. pub recompile: Option, @@ -922,11 +926,18 @@ pub enum Insn { } macro_rules! target_for_each_operand_impl { - ($self:expr, $visit_many:ident) => { + ($self:expr, $visit_one:ident, $visit_many:ident, $reborrow:ident) => { match $self { Target::SideExit(data) => { visit_many!(data.exit.stack); visit_many!(data.exit.locals); + if let Some(StackMap { stack, .. }) = $reborrow!(data.exit.stack_map) { + for entry in stack { + if let StackMapEntry::Opnd(opnd) = entry { + visit_one!(opnd); + } + } + } } Target::Block(edge) => { visit_many!(edge.args); @@ -956,17 +967,17 @@ macro_rules! for_each_operand_impl { Insn::Jz(target) | Insn::Label(target) | Insn::LeaJumpTarget { target, .. } => { - target_for_each_operand_impl!(target, $visit_many); + target_for_each_operand_impl!(target, $visit_one, $visit_many, $reborrow); } // `target` is behind a Box. `$reborrow` turns the box field into a `&`/`&mut Target` // matching the iterator, so the same operand-walk works for both. Insn::PatchPoint(data) => { - target_for_each_operand_impl!($reborrow!(data.target), $visit_many); + target_for_each_operand_impl!($reborrow!(data.target), $visit_one, $visit_many, $reborrow); } Insn::Joz(opnd, target) | Insn::Jonz(opnd, target) => { visit_one!(opnd); - target_for_each_operand_impl!(target, $visit_many); + target_for_each_operand_impl!(target, $visit_one, $visit_many, $reborrow); } Insn::BakeString(_) | @@ -1021,7 +1032,11 @@ macro_rules! for_each_operand_impl { // Option` matching the iterator, so the same operand-walk works for // both. if let Some(StackMap { stack, .. }) = $reborrow!(data.stack_map) { - visit_many!(stack); + for entry in stack { + if let StackMapEntry::Opnd(opnd) = entry { + visit_one!(opnd); + } + } } } // only iterate over preserved in the const iterator @@ -1484,15 +1499,18 @@ const JIT_FRAME_OFFSET_FROM_JIT_RETURN: usize = 1; /// | +-------------------------+ | /// v | opnds.first() | v /// +-------------------------+ -/// ^ | register spill slot 0 | ^ +/// ^ | allocator spill slot 0 | ^ /// | +-------------------------+ | /// | | ... | | stack_idx for "slot N" in StackState::stack_map_index_for_spill +/// num_spill_slots | +-------------------------+ | +/// v | allocator spill slot X | | +/// +-------------------------+ | +/// ^ | side-exit stack-map | | +/// | | capture slot 0 | | /// | +-------------------------+ | -/// num_spill_slots | | register spill slot N | v -/// | +-------------------------+ -/// | | ... | -/// | +-------------------------+ -/// v | register spill slot X | +/// num_side_exit_stack_map_slots | | ... | | +/// | +-------------------------+ | +/// v | capture slot X | v /// +-------------------------+ /// | FrameSetup align slot | if needed /// +-------------------------+ @@ -1505,17 +1523,21 @@ pub struct StackState { /// The number of stack slots needed by register allocator spills. pub(crate) num_spill_slots: usize, + + /// The maximum number of stack slots needed to capture side-exit stack-map + /// operands that cannot be encoded directly. + pub(crate) num_side_exit_stack_map_slots: usize, } impl StackState { /// Initialize an empty stack state. fn new() -> Self { - StackState { stack_base_idx: 0, num_spill_slots: 0 } + StackState { stack_base_idx: 0, num_spill_slots: 0, num_side_exit_stack_map_slots: 0 } } /// Initialize a stack state with a fixed number of reserved stack slots. fn new_with_stack_slots(stack_base_idx: usize) -> Self { - StackState { stack_base_idx, num_spill_slots: 0 } + StackState { stack_base_idx, num_spill_slots: 0, num_side_exit_stack_map_slots: 0 } } /// Reserve native stack slots for JITFrame storage and stack-allocated operands. @@ -1527,9 +1549,9 @@ impl StackState { } /// Return the total number of native stack slots used for the frame's - /// reserved data and register allocator spills. + /// reserved data, register allocator spills, and side-exit captures. pub(crate) fn stack_slot_count(&self) -> usize { - self.stack_base_idx + self.num_spill_slots + self.stack_base_idx + self.num_spill_slots + self.num_side_exit_stack_map_slots } /// Return the stack-map index for a VReg stored below StackState-managed @@ -1551,7 +1573,13 @@ impl StackState { /// Return a stack index for a register saved by handle_caller_saved_regs(). fn stack_idx_for_caller_saved_reg(&self, caller_saved_reg_idx: usize) -> usize { let frame_alignment_slots = self.stack_slot_count() % 2; - self.num_spill_slots + frame_alignment_slots + caller_saved_reg_idx + self.num_spill_slots + self.num_side_exit_stack_map_slots + frame_alignment_slots + caller_saved_reg_idx + } + + /// Return a stack index reserved for side-exit stack-map register capture. + fn stack_idx_for_side_exit_stack_map(&self, slot_idx: usize) -> usize { + assert!(slot_idx < self.num_side_exit_stack_map_slots); + self.num_spill_slots + slot_idx } /// Convert a stack index to the `disp` of the stack slot @@ -1572,12 +1600,10 @@ impl StackState { } /// Stack map to materialize Ruby stack slots from JIT-kept values. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Eq, Hash, PartialEq)] pub struct StackMap { /// Ruby stack slots to reconstruct if this frame is materialized. - /// Each operand must be either an immediate Ruby VALUE or a VReg whose - /// final register/spill location will be encoded after register allocation. - stack: Vec, + stack: Vec, /// Heap-allocated JITFrame whose trailing stack map storage receives the /// encoded entries once this CCall's register allocation is known. jit_frame: *const zjit_jit_frame, @@ -1586,6 +1612,21 @@ pub struct StackMap { frame_depth: usize, } +impl StackMap { + pub fn new(stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) -> Self { + Self { stack, jit_frame, frame_depth } + } +} + +/// Entry in a JITFrame stack map. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum StackMapEntry { + /// Immediate Ruby VALUE or VReg to materialize. + Opnd(Opnd), + /// Number of VM stack slots to skip when materializing across inlined frames. + Skip(usize), +} + /// Initial capacity for asm.insns vector const ASSEMBLER_INSNS_CAPACITY: usize = 256; @@ -2392,8 +2433,8 @@ impl Assembler // Build a set of VRegIds that can be referenced by JITFrame for materializing the VM stack let stack_vreg_ids: HashSet = if let Some(StackMap { stack, .. }) = &stack_map { - stack.iter().filter_map(|opnd| match opnd { - Opnd::VReg { idx, .. } => Some(*idx), + stack.iter().filter_map(|entry| match entry { + StackMapEntry::Opnd(Opnd::VReg { idx, .. }) => Some(*idx), _ => None, }).collect() } else { @@ -2437,18 +2478,28 @@ impl Assembler if let Some(StackMap { stack, jit_frame, frame_depth }) = stack_map { assert_eq!(unsafe { (*jit_frame).stack_size } as usize, stack.len()); - for (idx, stack_opnd) in stack.iter().enumerate() { - let entry = match stack_opnd { - Opnd::UImm(value) => { - let value = VALUE(*value as usize); + for (idx, stack_entry) in stack.iter().enumerate() { + let entry = match *stack_entry { + StackMapEntry::Opnd(Opnd::Value(value)) => { + // TODO: Investigate using a constant pool to track any value reference in the stack map + assert!(value.special_const_p(), "StackMap should only materialize immediate VALUEs, but got: {value:?}"); + value + } + StackMapEntry::Opnd(Opnd::UImm(value)) => { + let value = VALUE(value as usize); // TODO: Investigate using a constant pool to track any value reference in the stack map assert!(value.special_const_p(), "StackMap should only materialize immediate VALUEs, but got: {value:?}"); value } - Opnd::VReg { idx: vreg, .. } => { - let vreg_stack_index = match assignments[*vreg].expect("StackMap VReg should have an allocation") { + StackMapEntry::Skip(size) => { + let encoded = (size << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_SKIP_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap skip should not look like an immediate VALUE"); + VALUE(encoded) + } + StackMapEntry::Opnd(Opnd::VReg { idx: vreg, .. }) => { + let vreg_stack_index = match assignments[vreg].expect("StackMap VReg should have an allocation") { Allocation::Reg(_) | Allocation::Fixed(_) => { - let caller_saved_reg_idx = survivors.iter().position(|&survivor_id| survivor_id == *vreg).unwrap(); + let caller_saved_reg_idx = survivors.iter().position(|&survivor_id| survivor_id == vreg).unwrap(); let stack_idx = self.stack_state.stack_idx_for_caller_saved_reg(caller_saved_reg_idx); self.stack_state.stack_map_index_for_spill(stack_idx, frame_depth) } @@ -2462,7 +2513,7 @@ impl Assembler debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap VReg should not look like an immediate VALUE"); VALUE(encoded) } - _ => unreachable!("unexpected operand in StackMap: {stack_opnd:?}"), + _ => unreachable!("unexpected entry in StackMap: {stack_entry:?}"), }; unsafe { (*jit_frame.cast_mut()).stack.as_mut_ptr().add(idx).write(entry); } } @@ -2567,6 +2618,40 @@ impl Assembler } } + /// Return the maximum number of stack-map entries that any side exit needs + /// to copy into reserved native stack slots. + pub fn side_exit_stack_map_slots(&self, assignments: &[Option]) -> usize { + self.block_order().into_iter().fold(0, |max_slots, block_id| { + let block = &self.basic_blocks[block_id.0]; + block.insns.iter().fold(max_slots, |max_slots, insn| { + let slots = insn.target().map(|target| Self::side_exit_target_stack_map_slots(target, assignments)).unwrap_or(0); + max_slots.max(slots) + }) + }) + } + + fn side_exit_target_stack_map_slots(target: &Target, assignments: &[Option]) -> usize { + let Target::SideExit(data) = target else { + return 0; + }; + let Some(StackMap { stack, .. }) = &data.exit.stack_map else { + return 0; + }; + + stack.iter().filter(|entry| match entry { + StackMapEntry::Opnd(Opnd::Value(value)) => !value.special_const_p(), + StackMapEntry::Opnd(Opnd::UImm(value)) => !VALUE(*value as usize).special_const_p(), + StackMapEntry::Opnd(Opnd::VReg { idx, .. }) => { + matches!( + assignments[idx.to_usize()].expect("StackMap VReg should have an allocation"), + Allocation::Reg(_) | Allocation::Fixed(_) + ) + } + StackMapEntry::Opnd(Opnd::Reg(_)) => true, + _ => false, + }).count() + } + /// Walk every instruction and replace VReg operands with the physical /// register (or stack slot) from the allocation assignments. fn rewrite_instructions(&mut self, assignments: &[Option]) { @@ -2679,9 +2764,75 @@ impl Assembler /// Returns the exit code as a list of instructions to be appended after the main /// code is linearized and split. pub fn compile_exits(&mut self) -> Vec { + fn immediate_stack_map_value(opnd: Opnd) -> Option { + let value = match opnd { + Opnd::Value(value) => value, + Opnd::UImm(value) => VALUE(value as usize), + _ => unreachable!("unexpected immediate StackMap operand: {opnd:?}"), + }; + value.special_const_p().then_some(value) + } + + fn encode_stack_map_index(asm: &Assembler, stack_idx: usize, frame_depth: usize) -> VALUE { + let vreg_stack_index = asm.stack_state.stack_map_index_for_spill(stack_idx, frame_depth); + let encoded = (vreg_stack_index << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_VREG_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap VReg should not look like an immediate VALUE"); + VALUE(encoded) + } + + fn capture_stack_map_opnd(asm: &mut Assembler, opnd: Opnd, capture_idx: &mut usize, frame_depth: usize) -> VALUE { + let stack_idx = asm.stack_state.stack_idx_for_side_exit_stack_map(*capture_idx); + *capture_idx += 1; + let capture_slot = Opnd::Mem(Mem { + base: MemBase::Stack { stack_idx: stack_idx.try_into().unwrap(), num_bits: 64 }, + disp: 0, + num_bits: 64, + }); + let opnd = if matches!(opnd, Opnd::Reg(_)) { opnd.with_num_bits(64) } else { opnd }; + asm.store(capture_slot, opnd); + encode_stack_map_index(asm, stack_idx, frame_depth) + } + + fn compile_exit_stack_map(asm: &mut Assembler, stack_map: &StackMap) { + let StackMap { stack, jit_frame, frame_depth } = stack_map; + let jit_frame = *jit_frame; + assert_eq!(unsafe { (*jit_frame).stack_size } as usize, stack.len()); + + let mut capture_idx = 0; + for (idx, stack_entry) in stack.iter().enumerate() { + let entry = match *stack_entry { + StackMapEntry::Opnd(Opnd::Value(_) | Opnd::UImm(_)) => { + let StackMapEntry::Opnd(opnd) = *stack_entry else { unreachable!() }; + immediate_stack_map_value(opnd) + .unwrap_or_else(|| capture_stack_map_opnd(asm, opnd, &mut capture_idx, *frame_depth)) + } + StackMapEntry::Skip(size) => { + let encoded = (size << ZJIT_STACK_MAP_SHIFT) | ZJIT_STACK_MAP_SKIP_TAG as usize; + debug_assert!(!VALUE(encoded).special_const_p(), "encoded StackMap skip should not look like an immediate VALUE"); + VALUE(encoded) + } + StackMapEntry::Opnd(Opnd::Mem(Mem { base: MemBase::Stack { stack_idx, .. }, disp, .. })) => { + assert_eq!(disp, 0, "StackMap stack slot should not have a displacement"); + encode_stack_map_index(asm, stack_idx.to_usize(), *frame_depth) + } + StackMapEntry::Opnd(Opnd::Reg(_)) => { + let StackMapEntry::Opnd(opnd) = *stack_entry else { unreachable!() }; + capture_stack_map_opnd(asm, opnd, &mut capture_idx, *frame_depth) + } + _ => unreachable!("unexpected entry in SideExit StackMap: {stack_entry:?}"), + }; + unsafe { (*jit_frame.cast_mut()).stack.as_mut_ptr().add(idx).write(entry); } + } + + assert!(capture_idx <= asm.stack_state.num_side_exit_stack_map_slots); + asm_comment!(asm, "install side-exit JITFrame for caller depth {}", frame_depth); + let jit_frame_slot = Opnd::mem(64, NATIVE_BASE_PTR, -((*frame_depth as i32 + 1) * SIZEOF_VALUE_I32)); + asm.store(jit_frame_slot, Opnd::const_ptr(jit_frame)); + } + /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { - let SideExit { pc, stack, locals, iseq, .. } = exit; + let SideExit { pc, stack, locals, iseq, stack_map, .. } = exit; // Side exit blocks are not part of the CFG at the moment, // so we need to manually ensure that patchpoints get padded @@ -2712,6 +2863,10 @@ impl Assembler asm.store(Opnd::mem(64, SP, (-local_size_and_idx_to_ep_offset(locals.len(), idx) - 1) * SIZEOF_VALUE_I32), opnd); } } + + if let Some(stack_map) = stack_map { + compile_exit_stack_map(asm, stack_map); + } } /// Tear down the JIT frame and return to the interpreter. @@ -3846,9 +4001,9 @@ impl Assembler { /// HIR function currently reserves multiple native JITFrame slots, one per /// inlining depth, stack-map indexes must be encoded relative to the target /// frame's own cfp->jit_return. - pub fn stack_map(&mut self, stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) { + pub fn stack_map(&mut self, stack: Vec, jit_frame: *const zjit_jit_frame, frame_depth: usize) { assert!(self.stack_map.is_none()); - self.stack_map = Some(StackMap { stack, jit_frame, frame_depth }); + self.stack_map = Some(StackMap::new(stack, jit_frame, frame_depth)); } pub fn store(&mut self, dest: Opnd, src: Opnd) { diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 963a177dd8125a..e83ad9f2e7ad73 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1155,6 +1155,7 @@ impl Assembler { let (assignments, num_stack_slots) = trace_compile_phase("linear_scan", || asm.linear_scan(intervals.clone(), regs.len(), &preferred_registers)); asm.stack_state.num_spill_slots = num_stack_slots; + asm.stack_state.num_side_exit_stack_map_slots = asm.side_exit_stack_map_slots(&assignments); let stack_slot_count = asm.stack_state.stack_slot_count(); if stack_slot_count > Self::MAX_FRAME_STACK_SLOTS { return Err(CompileError::NativeStackTooLarge); @@ -1181,7 +1182,7 @@ impl Assembler { } // Update FrameSetup slot_count now that StackState knows the - // register allocator spill count. + // register allocator spill and side-exit capture counts. trace_compile_phase("count_stack_slots", || { for block in asm.basic_blocks.iter_mut() { for insn in block.insns.iter_mut() { @@ -1391,7 +1392,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } })); + let side_exit = Target::SideExit(Box::new(SideExitTarget { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], stack_map: None, recompile: None } })); asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 6a610e2c4235e4..0ad066c5e2bc24 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -21,7 +21,7 @@ use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Counter::{compile_time_ns, exit_compile_error}}; use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; -use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, SideExitTarget, Target, asm_ccall, asm_comment}; +use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, SideExitTarget, StackMap, StackMapEntry, Target, asm_ccall, asm_comment}; use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; use crate::hir::{BlockHandler, CCallVariadicData, CCallWithFrameData, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendDirectData, SendFallbackReason}; use crate::hir_type::{types, Type}; @@ -528,7 +528,7 @@ fn gen_function(cb: &mut CodeBlock, iseq: IseqPtr, version: IseqVersionRef, func Insn::InvokeBuiltin { .. } => SideExitReason::UnhandledHIRInvokeBuiltin, _ => SideExitReason::UnhandledHIRUnknown(insn_id), }; - gen_side_exit(&mut jit, &mut asm, &reason, None, &function.frame_state(last_snapshot)); + gen_side_exit(&mut jit, &mut asm, function, &reason, None, &function.frame_state(last_snapshot)); // Don't bother generating code after a side-exit. We won't run it. // TODO(max): Generate ud2 or equivalent. break; @@ -615,8 +615,8 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } Insn::Const { .. } => panic!("Unexpected Const in gen_insn: {insn}"), Insn::NewArray { elements, state } => gen_new_array(jit, asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewHash { elements, state } => gen_new_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), - Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), + Insn::NewHash { elements, state } => gen_new_hash(jit, asm, function, opnds!(elements), &function.frame_state(*state)), + Insn::NewRange { low, high, flag, state } => gen_new_range(jit, asm, function, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::NewRangeFixnum { low, high, flag, state } => gen_new_range_fixnum(asm, opnd!(low), opnd!(high), *flag, &function.frame_state(*state)), Insn::ArrayDup { val, state } => gen_array_dup(asm, opnd!(val), &function.frame_state(*state)), Insn::AdjustBounds { index, length } => gen_adjust_bounds(asm, opnd!(index), opnd!(length)), @@ -626,49 +626,50 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } Insn::ArrayPop { array, state } => gen_array_pop(asm, opnd!(array), &function.frame_state(*state)), Insn::ArrayLength { array } => gen_array_length(asm, opnd!(array)), - Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, opnd!(val), &function.frame_state(*state)), + Insn::ObjectAlloc { val, state } => gen_object_alloc(jit, asm, function, opnd!(val), &function.frame_state(*state)), &Insn::ObjectAllocClass { class, state } => gen_object_alloc_class(asm, class, &function.frame_state(state)), Insn::StringCopy { val, chilled, state } => gen_string_copy(asm, opnd!(val), *chilled, &function.frame_state(*state)), - Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, opnds!(strings), &function.frame_state(*state)), + Insn::StringConcat { strings, state } => gen_string_concat(jit, asm, function, opnds!(strings), &function.frame_state(*state)), &Insn::StringGetbyte { string, index } => gen_string_getbyte(asm, opnd!(string), opnd!(index)), Insn::StringSetbyteFixnum { string, index, value } => gen_string_setbyte_fixnum(asm, opnd!(string), opnd!(index), opnd!(value)), - Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), - Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppend { recv, other, state } => gen_string_append(jit, asm, function, opnd!(recv), opnd!(other), &function.frame_state(*state)), + Insn::StringAppendCodepoint { recv, other, state } => gen_string_append_codepoint(jit, asm, function, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::StringEqual { left, right } => gen_string_equal(asm, opnd!(left), opnd!(right)), Insn::StringIntern { val, state } => gen_intern(asm, opnd!(val), &function.frame_state(*state)), - Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, *opt, opnds!(values), &function.frame_state(*state)), + Insn::ToRegexp { opt, values, state } => gen_toregexp(jit, asm, function, *opt, opnds!(values), &function.frame_state(*state)), Insn::Param => unreachable!("block.insns should not have Insn::Param"), Insn::LoadArg { .. } => return Ok(()), // compiled in the LoadArg pre-pass above Insn::Snapshot { .. } => return Ok(()), // we don't need to do anything for this instruction at the moment - &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, cd, &function.frame_state(state), reason), - &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, cd, std::ptr::null(), &function.frame_state(state), reason), - &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: None, state, reason, .. } => gen_send_without_block(jit, asm, function, cd, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockIseq(blockiseq)), state, reason, .. } => gen_send(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::Send { cd, block: Some(BlockHandler::BlockArg), state, reason, .. } => gen_send(jit, asm, function, cd, std::ptr::null(), &function.frame_state(state), reason), + &Insn::SendForward { cd, blockiseq, state, reason, .. } => gen_send_forward(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), Insn::SendDirect(insn) => { let SendDirectData { cme, iseq, recv, args, kw_bits, block, state, .. } = &**insn; gen_send_iseq_direct( - cb, jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), + cb, jit, asm, + function, *cme, *iseq, opnd!(recv), opnds!(args), *kw_bits, &function.frame_state(*state), *block, ) } Insn::PushInlineFrame { cme, iseq, recv, args, blockiseq, state, .. } => { - no_output!(gen_push_inline_frame(jit, asm, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), *blockiseq)) + no_output!(gen_push_inline_frame(jit, asm, function, *cme, *iseq, opnd!(recv), opnds!(args), &function.frame_state(*state), *blockiseq)) }, Insn::PopInlineFrame { iseq, argc, state } => { no_output!(gen_pop_inline_frame(asm, *iseq, *argc, &function.frame_state(*state))) }, - &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), - &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), - Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), - Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), - Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), unsafe { &**bf }, *leaf, opnds!(args)), + &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, function, cd, blockiseq, &function.frame_state(state), reason), + &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, function, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, function, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), + Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, function, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, function, &function.frame_state(*state), unsafe { &**bf }, *leaf, opnds!(args)), &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), - Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), - Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumSub { left, right, state } => gen_fixnum_sub(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumMult { left, right, state } => gen_fixnum_mult(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), + Insn::FixnumDiv { left, right, state } => gen_fixnum_div(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(*state)), Insn::FloatAdd { recv, other, state } => gen_float_add(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::FloatSub { recv, other, state } => gen_float_sub(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), Insn::FloatMul { recv, other, state } => gen_float_mul(asm, opnd!(recv), opnd!(other), &function.frame_state(*state)), @@ -689,7 +690,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio // We only create FixnumLShift when we know the shift amount statically and it's in [0, // 63]. let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; - gen_fixnum_lshift(jit, asm, opnd!(left), shift_amount, &function.frame_state(state)) + gen_fixnum_lshift(jit, asm, function, opnd!(left), shift_amount, &function.frame_state(state)) } &Insn::FixnumRShift { left, right } => { // We only create FixnumRShift when we know the shift amount statically and it's in [0, @@ -697,13 +698,13 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio let shift_amount = function.type_of(right).fixnum_value().unwrap() as u64; gen_fixnum_rshift(asm, opnd!(left), shift_amount) } - &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), + &Insn::FixnumMod { left, right, state } => gen_fixnum_mod(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state)), &Insn::FixnumAref { recv, index } => gen_fixnum_aref(asm, opnd!(recv), opnd!(index)), &Insn::IsMethodCfunc { val, cd, cfunc, state } => gen_is_method_cfunc(asm, opnd!(val), cd, cfunc, &function.frame_state(state)), &Insn::IsBitEqual { left, right } => gen_is_bit_equal(asm, opnd!(left), opnd!(right)), &Insn::IsBitNotEqual { left, right } => gen_is_bit_not_equal(asm, opnd!(left), opnd!(right)), &Insn::BoxBool { val } => gen_box_bool(asm, opnd!(val)), - &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, opnd!(val), &function.frame_state(state)), + &Insn::BoxFixnum { val, state } => gen_box_fixnum(jit, asm, function, opnd!(val), &function.frame_state(state)), &Insn::UnboxFixnum { val } => gen_unbox_fixnum(asm, opnd!(val)), Insn::Test { val } => gen_test(asm, opnd!(val)), Insn::RefineType { val, .. } => opnd!(val), @@ -713,55 +714,55 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio } &Insn::GuardType { val, guard_type, state, recompile } => { let val_type = function.type_of(val); - gen_guard_type(jit, asm, opnd!(val), val_type, guard_type, recompile, &function.frame_state(state)) + gen_guard_type(jit, asm, function, opnd!(val), val_type, guard_type, recompile, &function.frame_state(state)) } - &Insn::GuardBitEquals { val, expected, ref reason, state, recompile } => gen_guard_bit_equals(jit, asm, opnd!(val), expected, **reason, recompile, &function.frame_state(state)), - &Insn::GuardAnyBitSet { val, mask, ref reason, state, recompile, .. } => gen_guard_any_bit_set(jit, asm, opnd!(val), mask, **reason, recompile, &function.frame_state(state)), - &Insn::GuardNoBitsSet { val, mask, ref reason, state, .. } => gen_guard_no_bits_set(jit, asm, opnd!(val), mask, **reason, &function.frame_state(state)), - &Insn::GuardLess { left, right, ref reason, state } => gen_guard_less(jit, asm, opnd!(left), opnd!(right), **reason, &function.frame_state(state)), - &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), - Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), + &Insn::GuardBitEquals { val, expected, ref reason, state, recompile } => gen_guard_bit_equals(jit, asm, function, opnd!(val), expected, **reason, recompile, &function.frame_state(state)), + &Insn::GuardAnyBitSet { val, mask, ref reason, state, recompile, .. } => gen_guard_any_bit_set(jit, asm, function, opnd!(val), mask, **reason, recompile, &function.frame_state(state)), + &Insn::GuardNoBitsSet { val, mask, ref reason, state, .. } => gen_guard_no_bits_set(jit, asm, function, opnd!(val), mask, **reason, &function.frame_state(state)), + &Insn::GuardLess { left, right, ref reason, state } => gen_guard_less(jit, asm, function, opnd!(left), opnd!(right), **reason, &function.frame_state(state)), + &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state)), + Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, function, invariant, &function.frame_state(*state))), Insn::CCall { cfunc, recv, args, name, owner: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), Insn::CCallWithFrame(insn) => { let CCallWithFrameData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; - gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + gen_ccall_with_frame(jit, asm, function, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) } Insn::CCallVariadic(insn) => { let CCallVariadicData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; - gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + gen_ccall_variadic(jit, asm, function, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) } Insn::GetIvar { self_val, id, ic, state } => gen_getivar(asm, opnd!(self_val), *id, *ic, &function.frame_state(*state)), - Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, *id, opnd!(val), &function.frame_state(*state))), - Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, *id, &function.frame_state(*state)), + Insn::SetGlobal { id, val, state } => no_output!(gen_setglobal(jit, asm, function, *id, opnd!(val), &function.frame_state(*state))), + Insn::GetGlobal { id, state } => gen_getglobal(jit, asm, function, *id, &function.frame_state(*state)), &Insn::IsBlockParamModified { flags } => gen_is_block_param_modified(asm, opnd!(flags)), - &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, ep_offset, level, &function.frame_state(state)), + &Insn::GetBlockParam { ep_offset, level, state } => gen_getblockparam(jit, asm, function, ep_offset, level, &function.frame_state(state)), &Insn::SetLocal { val, ep_offset, level, .. } => no_output!(gen_setlocal(asm, opnd!(val), function.type_of(val), ep_offset, level)), - Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), - Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, *ic, &function.frame_state(*state)), - Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, *id, *ic, &function.frame_state(*state)), - Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, *id, opnd!(val), *ic, &function.frame_state(*state))), - Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), + Insn::GetConstant { klass, id, allow_nil, state } => gen_getconstant(jit, asm, function, opnd!(klass), *id, opnd!(allow_nil), &function.frame_state(*state)), + Insn::GetConstantPath { ic, state } => gen_get_constant_path(jit, asm, function, *ic, &function.frame_state(*state)), + Insn::GetClassVar { id, ic, state } => gen_getclassvar(jit, asm, function, *id, *ic, &function.frame_state(*state)), + Insn::SetClassVar { id, val, ic, state } => no_output!(gen_setclassvar(jit, asm, function, *id, opnd!(val), *ic, &function.frame_state(*state))), + Insn::SetIvar { self_val, id, ic, val, state } => no_output!(gen_setivar(jit, asm, function, opnd!(self_val), *id, *ic, opnd!(val), &function.frame_state(*state))), Insn::FixnumBitCheck { val, index } => gen_fixnum_bit_check(asm, opnd!(val), *index), - Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, reason, *recompile, &function.frame_state(*state))), - Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, *value_type, &function.frame_state(*state)), + Insn::SideExit { state, reason, recompile } => no_output!(gen_side_exit(jit, asm, function, reason, *recompile, &function.frame_state(*state))), + Insn::PutSpecialObject { value_type, state } => gen_putspecialobject(jit, asm, function, *value_type, &function.frame_state(*state)), Insn::AnyToString { val, str, state } => gen_anytostring(asm, opnd!(val), opnd!(str), &function.frame_state(*state)), - Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), - Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), + Insn::Defined { op_type, obj, pushval, v, lep_level, state } => gen_defined(jit, asm, function, *op_type, *obj, *pushval, opnd!(v), *lep_level, &function.frame_state(*state)), + Insn::CheckMatch { target, pattern, flag, state } => gen_checkmatch(jit, asm, function, opnd!(target), opnd!(pattern), *flag, &function.frame_state(*state)), Insn::GetSpecialSymbol { symbol_type, state } => gen_getspecial_symbol(asm, *symbol_type, &function.frame_state(*state)), Insn::GetSpecialNumber { nth, state } => gen_getspecial_number(asm, *nth, &function.frame_state(*state)), &Insn::IncrCounter(counter) => no_output!(gen_incr_counter(asm, counter)), Insn::IncrCounterPtr { counter_ptr } => no_output!(gen_incr_counter_ptr(asm, *counter_ptr)), - &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, &function.frame_state(state))), + &Insn::CheckInterrupts { state } => no_output!(gen_check_interrupts(jit, asm, function, &function.frame_state(state))), Insn::BreakPoint => no_output!(asm.breakpoint()), Insn::Unreachable => no_output!(asm.abort()), &Insn::HashDup { val, state } => { gen_hash_dup(asm, opnd!(val), &function.frame_state(state)) }, - &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, opnd!(hash), opnd!(key), &function.frame_state(state)) }, - &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, + &Insn::HashAref { hash, key, state } => { gen_hash_aref(jit, asm, function, opnd!(hash), opnd!(key), &function.frame_state(state)) }, + &Insn::HashAset { hash, key, val, state } => { no_output!(gen_hash_aset(jit, asm, function, opnd!(hash), opnd!(key), opnd!(val), &function.frame_state(state))) }, &Insn::ArrayPush { array, val, state } => { no_output!(gen_array_push(asm, opnd!(array), opnd!(val), &function.frame_state(state))) }, - &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, opnd!(val), &function.frame_state(state)) }, - &Insn::ToArray { val, state } => { gen_to_array(jit, asm, opnd!(val), &function.frame_state(state)) }, + &Insn::ToNewArray { val, state } => { gen_to_new_array(jit, asm, function, opnd!(val), &function.frame_state(state)) }, + &Insn::ToArray { val, state } => { gen_to_array(jit, asm, function, opnd!(val), &function.frame_state(state)) }, &Insn::DefinedIvar { self_val, id, pushval, .. } => { gen_defined_ivar(asm, opnd!(self_val), id, pushval) }, - &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state))) }, + &Insn::ArrayExtend { left, right, state } => { no_output!(gen_array_extend(jit, asm, function, opnd!(left), opnd!(right), &function.frame_state(state))) }, Insn::LoadPC => gen_load_pc(asm), Insn::LoadEC => gen_load_ec(), Insn::LoadSP => gen_load_sp(), @@ -771,13 +772,13 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::StoreField { recv, id, offset, val } => no_output!(gen_store_field(asm, opnd!(recv), id, offset, opnd!(val), function.type_of(val))), &Insn::WriteBarrier { recv, val } => no_output!(gen_write_barrier(jit, asm, opnd!(recv), opnd!(val), function.type_of(val))), &Insn::IsBlockGiven { lep } => gen_is_block_given(asm, opnd!(lep)), - Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, opnds!(elements), opnd!(target), &function.frame_state(*state)), - Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), - &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, ary, opnd!(target), &function.frame_state(state)), - Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, opnds!(elements), &function.frame_state(*state)), + Insn::ArrayInclude { elements, target, state } => gen_array_include(jit, asm, function, opnds!(elements), opnd!(target), &function.frame_state(*state)), + Insn::ArrayPackBuffer { elements, fmt, buffer, state } => gen_array_pack_buffer(jit, asm, function, opnds!(elements), opnd!(fmt), (*buffer).map(|buffer| opnd!(buffer)), &function.frame_state(*state)), + &Insn::DupArrayInclude { ary, target, state } => gen_dup_array_include(jit, asm, function, ary, opnd!(target), &function.frame_state(state)), + Insn::ArrayHash { elements, state } => gen_opt_newarray_hash(jit, asm, function, opnds!(elements), &function.frame_state(*state)), &Insn::IsA { val, class } => gen_is_a(jit, asm, opnd!(val), opnd!(class)), - &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, opnds!(elements), &function.frame_state(state)), - &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMax { ref elements, state } => gen_array_max(jit, asm, function, opnds!(elements), &function.frame_state(state)), + &Insn::ArrayMin { ref elements, state } => gen_array_min(jit, asm, function, opnds!(elements), &function.frame_state(state)), &Insn::Throw { state, .. } => return Err(state), &Insn::CondBranch { .. } | &Insn::Jump { .. } | Insn::Entries { .. } => unreachable!(), @@ -810,7 +811,7 @@ fn gen_get_ep(asm: &mut Assembler, level: u32) -> Opnd { ep_opnd } -fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { +fn gen_defined(jit: &JITState, asm: &mut Assembler, function: &Function, op_type: usize, obj: VALUE, pushval: VALUE, tested_value: Opnd, lep_level: u32, state: &FrameState) -> Opnd { match op_type as defined_type { DEFINED_YIELD => { // `lep_level` was precomputed at HIR construction so we can materialize the local EP @@ -823,7 +824,7 @@ fn gen_defined(jit: &JITState, asm: &mut Assembler, op_type: usize, obj: VALUE, } _ => { // Save the PC and SP because the callee may allocate or call #respond_to? - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // TODO: Inline the cases for each op_type // Call vm_defined(ec, reg_cfp, op_type, obj, v) @@ -877,13 +878,13 @@ fn gen_is_block_param_modified(asm: &mut Assembler, flags: Opnd) -> Opnd { /// Get the block parameter as a Proc, write it to the environment, /// and mark the flag as modified. -fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { +fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, function: &Function, ep_offset: u32, level: u32, state: &FrameState) -> Opnd { gen_prepare_leaf_call_with_gc(asm, state); // Bail out if write barrier is required. let ep = gen_get_ep(asm, level); let flags = Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * (VM_ENV_DATA_INDEX_FLAGS as i32)); asm.test(flags, VM_ENV_FLAG_WB_REQUIRED.into()); - asm.jnz(jit, side_exit(jit, state, SideExitReason::BlockParamWbRequired)); + asm.jnz(jit, side_exit(jit, function, state, SideExitReason::BlockParamWbRequired)); // Convert block handler to Proc. let block_handler = asm.load(Opnd::mem(VALUE_BITS, ep, SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL)); @@ -903,36 +904,36 @@ fn gen_getblockparam(jit: &mut JITState, asm: &mut Assembler, ep_offset: u32, le asm.load(Opnd::mem(VALUE_BITS, ep, offset)) } -fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { +fn gen_guard_less(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, reason: SideExitReason, state: &FrameState) -> Opnd { asm.cmp(left, right); - asm.jge(jit, side_exit(jit, state, reason)); + asm.jge(jit, side_exit(jit, function, state, reason)); left } -fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { +fn gen_guard_greater_eq(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, state: &FrameState) -> Opnd { asm.cmp(left, right); - asm.jl(jit, side_exit(jit, state, SideExitReason::GuardGreaterEq)); + asm.jl(jit, side_exit(jit, function, state, SideExitReason::GuardGreaterEq)); left } -fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { +fn gen_get_constant_path(jit: &JITState, asm: &mut Assembler, function: &Function, ic: *const iseq_inline_constant_cache, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_opt_getconstant_path(ec: EcPtr, cfp: CfpPtr, ic: *const iseq_inline_constant_cache) -> VALUE; } // Anything could be called on const_missing - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_opt_getconstant_path, EC, CFP, Opnd::const_ptr(ic)) } -fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { +fn gen_getconstant(jit: &mut JITState, asm: &mut Assembler, function: &Function, klass: Opnd, id: ID, allow_nil: Opnd, state: &FrameState) -> Opnd { unsafe extern "C" { fn rb_vm_get_ev_const(ec: EcPtr, klass: VALUE, id: ID, allow_nil: VALUE) -> VALUE; } // Constant lookup can raise and run arbitrary Ruby code via const_missing. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_get_ev_const, EC, klass, id.0.into(), allow_nil) } @@ -943,7 +944,7 @@ fn gen_fixnum_bit_check(asm: &mut Assembler, val: Opnd, index: u8) -> Opnd { asm.csel_z(Qtrue.into(), Qfalse.into()) } -fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec) -> lir::Opnd { +fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState, bf: &rb_builtin_function, leaf: bool, args: Vec) -> lir::Opnd { // +2 for ec, self assert!(bf.argc + 2 <= C_ARG_OPNDS.len() as i32, "gen_invokebuiltin should not be called for builtin function {} with too many arguments: {}", @@ -953,7 +954,7 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf gen_prepare_leaf_call_with_gc(asm, state); } else { // Anything can happen inside builtin functions - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); } let mut cargs = vec![EC]; @@ -964,9 +965,9 @@ fn gen_invokebuiltin(jit: &JITState, asm: &mut Assembler, state: &FrameState, bf } /// Record a patch point that should be invalidated on a given invariant -fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, invariant: &Invariant, state: &FrameState) { +fn gen_patch_point(jit: &mut JITState, asm: &mut Assembler, function: &Function, invariant: &Invariant, state: &FrameState) { let invariant = *invariant; - let exit = build_side_exit(jit, state); + let exit = build_side_exit(jit, function, state); // Let compile_exits compile a side exit. Let scratch_split lower it with split_patch_point. asm.patch_point(Target::SideExit(Box::new(SideExitTarget { exit, reason: PatchPoint(invariant) })), invariant, jit.version); @@ -1018,6 +1019,7 @@ pub fn split_patch_point(asm: &mut Assembler, target: &Target, invariant: Invari fn gen_ccall_with_frame( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cfunc: *const u8, name: ID, recv: Opnd, @@ -1027,7 +1029,7 @@ fn gen_ccall_with_frame( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::non_variadic_cfunc_optimized_send_count); - gen_stack_overflow_check(jit, asm, state, state.stack_size()); + gen_stack_overflow_check(jit, asm, function, state, state.stack_size()); let args_with_recv_len = args.len() + 1; if args_with_recv_len > C_ARG_OPNDS.len() { @@ -1039,7 +1041,7 @@ fn gen_ccall_with_frame( // to account for the receiver and arguments (and block arguments if any) gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, caller_stack_size); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); gen_spill_locals(jit, asm, state); let block_handler_specval = if let Some(BlockHandler::BlockIseq(block_iseq)) = block { @@ -1112,6 +1114,7 @@ fn gen_block_handler_specval(asm: &mut Assembler, blockiseq: IseqPtr) -> lir::Op fn gen_ccall_variadic( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cfunc: *const u8, name: ID, recv: Opnd, @@ -1121,7 +1124,7 @@ fn gen_ccall_variadic( state: &FrameState, ) -> lir::Opnd { gen_incr_counter(asm, Counter::variadic_cfunc_optimized_send_count); - gen_stack_overflow_check(jit, asm, state, state.stack_size()); + gen_stack_overflow_check(jit, asm, function, state, state.stack_size()); let args_with_recv_len = args.len() + 1; @@ -1133,7 +1136,7 @@ fn gen_ccall_variadic( // to account for the receiver and arguments (like gen_ccall_with_frame does) gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, caller_stack_size); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); gen_spill_locals(jit, asm, state); let block_handler_specval = if let Some(BlockHandler::BlockIseq(blockiseq)) = block { @@ -1188,9 +1191,9 @@ fn gen_getivar(asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_i } /// Emit an uncached instance variable store -fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { +fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, function: &Function, recv: Opnd, id: ID, ic: *const iseq_inline_iv_cache_entry, val: Opnd, state: &FrameState) { // Setting an ivar can raise FrozenError, so we need proper frame state for exception handling. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); if ic.is_null() { asm_ccall!(asm, rb_ivar_set, recv, id.0.into(), val); } else { @@ -1199,20 +1202,20 @@ fn gen_setivar(jit: &mut JITState, asm: &mut Assembler, recv: Opnd, id: ID, ic: } } -fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_getclassvar(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_getclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), Opnd::const_ptr(ic)) } -fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_setclassvar(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, val: Opnd, ic: *const iseq_inline_cvar_cache_entry, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_setclassvariable, VALUE::from(state.iseq).into(), CFP, id.0.into(), val, Opnd::const_ptr(ic)); } /// Look up global variables -fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, state: &FrameState) -> Opnd { +fn gen_getglobal(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, state: &FrameState) -> Opnd { // `Warning` module's method `warn` can be called when reading certain global variables - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_gvar_get, id.0.into()) } @@ -1225,25 +1228,25 @@ fn gen_intern(asm: &mut Assembler, val: Opnd, state: &FrameState) -> Opnd { } /// Set global variables -fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, id: ID, val: Opnd, state: &FrameState) { +fn gen_setglobal(jit: &mut JITState, asm: &mut Assembler, function: &Function, id: ID, val: Opnd, state: &FrameState) { // When trace_var is used, setting a global variable can cause exceptions - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_gvar_set, id.0.into(), val); } /// Side-exit into the interpreter -fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, reason: &SideExitReason, recompile: Option, state: &FrameState) { - asm.jmp(side_exit_with_recompile(jit, state, *reason, recompile)); +fn gen_side_exit(jit: &mut JITState, asm: &mut Assembler, function: &Function, reason: &SideExitReason, recompile: Option, state: &FrameState) { + asm.jmp(side_exit_with_recompile(jit, function, state, *reason, recompile)); } /// Emit a special object lookup -fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, value_type: SpecialObjectType, state: &FrameState) -> Opnd { +fn gen_putspecialobject(jit: &JITState, asm: &mut Assembler, function: &Function, value_type: SpecialObjectType, state: &FrameState) -> Opnd { // rb_vm_get_special_object for CBASE/CONST_BASE can call rb_singleton_class, // which allocates (may trigger GC) and can raise TypeError on non-class // receivers (e.g. `123.instance_eval { Const = 1 }`). Treat as non-leaf so // the PC is saved for GC and stack/locals are spilled for rescue. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // Get the EP of the current CFP and load it into a register let ep_opnd = Opnd::mem(64, CFP, RUBY_OFFSET_CFP_EP); @@ -1287,7 +1290,7 @@ fn gen_getspecial_number(asm: &mut Assembler, nth: u64, state: &FrameState) -> O asm_ccall!(asm, rb_reg_nth_match, Opnd::Imm((nth >> 1).try_into().unwrap()), backref) } -fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // Check for interrupts // see RUBY_VM_CHECK_INTS(ec) macro asm_comment!(asm, "RUBY_VM_CHECK_INTS(ec)"); @@ -1295,7 +1298,7 @@ fn gen_check_interrupts(jit: &mut JITState, asm: &mut Assembler, state: &FrameSt // signal_exec, or rb_postponed_job_flush. let interrupt_flag = asm.load(Opnd::mem(32, EC, RUBY_OFFSET_EC_INTERRUPT_FLAG)); asm.test(interrupt_flag, interrupt_flag); - asm.jnz(jit, side_exit(jit, state, SideExitReason::Interrupt)); + asm.jnz(jit, side_exit(jit, function, state, SideExitReason::Interrupt)); } fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { @@ -1303,13 +1306,13 @@ fn gen_hash_dup(asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd asm_ccall!(asm, rb_hash_resurrect, val) } -fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_hash_aref(jit: &mut JITState, asm: &mut Assembler, function: &Function, hash: Opnd, key: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_hash_aref, hash, key) } -fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_hash_aset(jit: &mut JITState, asm: &mut Assembler, function: &Function, hash: Opnd, key: Opnd, val: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_hash_aset, hash, key, val); } @@ -1318,13 +1321,13 @@ fn gen_array_push(asm: &mut Assembler, array: Opnd, val: Opnd, state: &FrameStat asm_ccall!(asm, rb_ary_push, array, val); } -fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_to_new_array(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qtrue), val) } -fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, val: Opnd, state: &FrameState) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_to_array(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: Opnd, state: &FrameState) -> lir::Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_vm_splat_array, Opnd::Value(Qfalse), val) } @@ -1332,11 +1335,11 @@ fn gen_defined_ivar(asm: &mut Assembler, self_val: Opnd, id: ID, pushval: VALUE) asm_ccall!(asm, rb_zjit_defined_ivar, self_val, id.0.into(), Opnd::Value(pushval)) } -fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { +fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, function: &Function, target: Opnd, pattern: Opnd, flag: u32, state: &FrameState) -> lir::Opnd { // rb_vm_check_match is not leaf unless flag is VM_CHECKMATCH_TYPE_WHEN. // See also: leafness_of_checkmatch() and check_match() if flag != VM_CHECKMATCH_TYPE_WHEN { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); } unsafe extern "C" { @@ -1346,8 +1349,8 @@ fn gen_checkmatch(jit: &JITState, asm: &mut Assembler, target: Opnd, pattern: Op asm_ccall!(asm, rb_vm_check_match, EC, target, pattern, flag.into()) } -fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, left: Opnd, right: Opnd, state: &FrameState) { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_array_extend(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: Opnd, right: Opnd, state: &FrameState) { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_ary_concat, left, right); } @@ -1465,6 +1468,7 @@ fn gen_param(asm: &mut Assembler, _idx: usize) -> lir::Opnd { fn gen_send( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1472,7 +1476,7 @@ fn gen_send( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_send(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -1488,6 +1492,7 @@ fn gen_send( fn gen_send_forward( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1495,7 +1500,7 @@ fn gen_send_forward( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { @@ -1512,13 +1517,14 @@ fn gen_send_forward( fn gen_send_without_block( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, state: &FrameState, reason: SendFallbackReason, ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call #{} with dynamic dispatch", ruby_call_method_name(cd)); unsafe extern "C" { fn rb_vm_opt_send_without_block(ec: EcPtr, cfp: CfpPtr, cd: VALUE) -> VALUE; @@ -1536,6 +1542,7 @@ fn gen_send_without_block( fn gen_push_inline_frame( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cme: *const rb_callable_method_entry_t, iseq: IseqPtr, recv: Opnd, @@ -1545,7 +1552,7 @@ fn gen_push_inline_frame( ) { let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); - gen_stack_overflow_check(jit, asm, state, stack_growth); + gen_stack_overflow_check(jit, asm, function, state, stack_growth); // Save cfp->pc and cfp->sp for the caller frame. // Cannot use gen_prepare_non_leaf_call because we need special SP math. @@ -1554,7 +1561,6 @@ fn gen_push_inline_frame( gen_save_sp(asm, stack_size); gen_spill_locals(jit, asm, state); - gen_spill_stack(jit, asm, state); // This mirrors vm_caller_setup_arg_block() for the `blockiseq != NULL` case. // The HIR specialization guards ensure we will only reach here for literal blocks, @@ -1670,6 +1676,7 @@ fn gen_send_iseq_direct( cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, + function: &Function, cme: *const rb_callable_method_entry_t, iseq: IseqPtr, recv: Opnd, @@ -1682,16 +1689,17 @@ fn gen_send_iseq_direct( let local_size = unsafe { get_iseq_body_local_table_size(iseq) }.to_usize(); let stack_growth = state.stack_size() + local_size + unsafe { get_iseq_body_stack_max(iseq) }.to_usize(); - gen_stack_overflow_check(jit, asm, state, stack_growth); + gen_stack_overflow_check(jit, asm, function, state, stack_growth); // Save cfp->pc and cfp->sp for the caller frame // Can't use gen_prepare_non_leaf_call because we need special SP math. let stack_size = state.stack().len() - args.len() - 1; // -1 for receiver - let jit_frame = gen_write_jit_frame(asm, state, stack_size); + let stack_map = build_stack_map(jit, function, &state.with_stack_size(stack_size)); + let jit_frame = gen_write_jit_frame(asm, state, stack_map.len()); gen_save_sp(asm, stack_size); gen_spill_locals(jit, asm, state); - gen_stack_map(jit, asm, state, stack_size, jit_frame); + asm.stack_map(stack_map, jit_frame, state.depth); // This mirrors vm_caller_setup_arg_block() in for the `blockiseq != NULL` case. // The HIR specialization guards ensure we will only reach here for literal blocks, @@ -1820,13 +1828,14 @@ fn gen_send_iseq_direct( fn gen_invokeblock( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, state: &FrameState, reason: SendFallbackReason, ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call invokeblock"); unsafe extern "C" { @@ -1844,6 +1853,7 @@ fn gen_invokeblock( fn gen_invokeblock_ifunc( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, block_handler: Opnd, args: Vec, @@ -1851,7 +1861,7 @@ fn gen_invokeblock_ifunc( ) -> lir::Opnd { let _ = cd; // cd is not needed for the direct call - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); // Push args to memory so we can pass argv pointer let argv_ptr = gen_push_opnds(jit, asm, &args); @@ -1876,12 +1886,13 @@ fn gen_invokeblock_ifunc( fn gen_invokeproc( jit: &mut JITState, asm: &mut Assembler, + function: &Function, recv: Opnd, args: Vec, kw_splat: bool, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call invokeproc"); @@ -1903,6 +1914,7 @@ fn gen_invokeproc( fn gen_invokesuper( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1910,7 +1922,7 @@ fn gen_invokesuper( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call super with dynamic dispatch"); unsafe extern "C" { fn rb_vm_invokesuper(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -1926,6 +1938,7 @@ fn gen_invokesuper( fn gen_invokesuperforward( jit: &mut JITState, asm: &mut Assembler, + function: &Function, cd: *const rb_call_data, blockiseq: IseqPtr, state: &FrameState, @@ -1933,7 +1946,7 @@ fn gen_invokesuperforward( ) -> lir::Opnd { gen_incr_send_fallback_counter(asm, reason); - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); asm_comment!(asm, "call super with dynamic dispatch (forwarding)"); unsafe extern "C" { fn rb_vm_invokesuperforward(ec: EcPtr, cfp: CfpPtr, cd: VALUE, blockiseq: IseqPtr) -> VALUE; @@ -2058,11 +2071,12 @@ fn gen_array_ptr(asm: &mut Assembler, array: Opnd) -> lir::Opnd { fn gen_opt_newarray_hash( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { // `Array#hash` will hash the elements of the array. - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2085,10 +2099,11 @@ fn gen_opt_newarray_hash( fn gen_array_max( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); @@ -2111,10 +2126,11 @@ fn gen_array_max( fn gen_array_min( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: u32 = elements.len().try_into().expect("Unable to fit length of elements into u32"); @@ -2136,11 +2152,12 @@ fn gen_array_min( fn gen_array_include( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, target: Opnd, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2163,12 +2180,13 @@ fn gen_array_include( fn gen_array_pack_buffer( jit: &JITState, asm: &mut Assembler, + function: &Function, elements: Vec, fmt: Opnd, buffer: Option, state: &FrameState, ) -> lir::Opnd { - gen_prepare_fallback_call(jit, asm, state); + gen_prepare_fallback_call(jit, asm, function, state); let array_len: c_long = elements.len().try_into().expect("Unable to fit length of elements into c_long"); @@ -2195,11 +2213,12 @@ fn gen_array_pack_buffer( fn gen_dup_array_include( jit: &JITState, asm: &mut Assembler, + function: &Function, ary: VALUE, target: Opnd, state: &FrameState, ) -> lir::Opnd { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); unsafe extern "C" { fn rb_vm_opt_duparray_include_p(ec: EcPtr, ary: VALUE, target: VALUE) -> VALUE; @@ -2264,6 +2283,7 @@ fn gen_is_a(jit: &mut JITState, asm: &mut Assembler, obj: Opnd, class: Opnd) -> fn gen_new_hash( jit: &mut JITState, asm: &mut Assembler, + function: &Function, elements: Vec, state: &FrameState, ) -> lir::Opnd { @@ -2283,7 +2303,7 @@ fn gen_new_hash( asm.store(Opnd::mem(VALUE_BITS, hash, RUBY_OFFSET_RHASH_IFNONE), Qnil.into()); hash } else { - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); let argv = gen_push_opnds(jit, asm, &elements); asm_ccall!(asm, rb_hash_new_with_bulk_insert, elements.len().into(), argv) @@ -2294,13 +2314,14 @@ fn gen_new_hash( fn gen_new_range( jit: &JITState, asm: &mut Assembler, + function: &Function, low: lir::Opnd, high: lir::Opnd, flag: RangeType, state: &FrameState, ) -> lir::Opnd { // Sometimes calls `low.<=>(high)` - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); // Call rb_range_new(low, high, flag) asm_ccall!(asm, rb_range_new, low, high, (flag as i32).into()) @@ -2317,9 +2338,9 @@ fn gen_new_range_fixnum( asm_ccall!(asm, rb_range_new, low, high, (flag as i64).into()) } -fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_object_alloc(jit: &JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, state: &FrameState) -> lir::Opnd { // Allocating an object from an unknown class is non-leaf; see doc for `ObjectAlloc`. - gen_prepare_non_leaf_call(jit, asm, state); + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_obj_alloc, val) } @@ -2391,25 +2412,25 @@ fn gen_return(asm: &mut Assembler, val: lir::Opnd) { } /// Compile Fixnum + Fixnum -fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_add(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Add left + right and test for overflow let left_untag = asm.sub(left, Opnd::Imm(1)); let out_val = asm.add(left_untag, right); - asm.jo(jit, side_exit(jit, state, FixnumAddOverflow)); + asm.jo(jit, side_exit(jit, function, state, FixnumAddOverflow)); out_val } /// Compile Fixnum - Fixnum -fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_sub(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Subtract left - right and test for overflow let val_untag = asm.sub(left, right); - asm.jo(jit, side_exit(jit, state, FixnumSubOverflow)); + asm.jo(jit, side_exit(jit, function, state, FixnumSubOverflow)); asm.add(val_untag, Opnd::Imm(1)) } /// Compile Fixnum * Fixnum -fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Do some bitwise gymnastics to handle tag bits // x * y is translated to (x >> 1) * (y - 1) + 1 let left_untag = asm.rshift(left, Opnd::UImm(1)); @@ -2417,17 +2438,17 @@ fn gen_fixnum_mult(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, rig let out_val = asm.mul(left_untag, right_untag); // Test for overflow - asm.jo_mul(jit, side_exit(jit, state, FixnumMultOverflow)); + asm.jo_mul(jit, side_exit(jit, function, state, FixnumMultOverflow)); asm.add(out_val, Opnd::UImm(1)) } /// Compile Fixnum / Fixnum -fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_div(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { gen_prepare_leaf_call_with_gc(asm, state); // Side exit if rhs is 0 asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); - asm.je(jit, side_exit(jit, state, FixnumDivByZero)); + asm.je(jit, side_exit(jit, function, state, FixnumDivByZero)); asm_ccall!(asm, rb_jit_fix_div_fix, left, right) } @@ -2520,14 +2541,14 @@ fn gen_fixnum_xor(asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd) -> lir } /// Compile Fixnum << Fixnum -fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_lshift(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, shift_amount: u64, state: &FrameState) -> lir::Opnd { // Shift amount is known statically to be in the range [0, 63] assert!(shift_amount < 64); let in_val = asm.sub(left, Opnd::UImm(1)); // Drop tag bit let out_val = asm.lshift(in_val, shift_amount.into()); let unshifted = asm.rshift(out_val, shift_amount.into()); asm.cmp(in_val, unshifted); - asm.jne(jit, side_exit(jit, state, FixnumLShiftOverflow)); + asm.jne(jit, side_exit(jit, function, state, FixnumLShiftOverflow)); // Re-tag the output value let out_val = asm.add(out_val, 1.into()); out_val @@ -2542,10 +2563,10 @@ fn gen_fixnum_rshift(asm: &mut Assembler, left: lir::Opnd, shift_amount: u64) -> asm.or(result, 1.into()) } -fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_fixnum_mod(jit: &mut JITState, asm: &mut Assembler, function: &Function, left: lir::Opnd, right: lir::Opnd, state: &FrameState) -> lir::Opnd { // Check for left % 0, which raises ZeroDivisionError asm.cmp(right, Opnd::from(VALUE::fixnum_from_usize(0))); - asm.je(jit, side_exit(jit, state, FixnumModByZero)); + asm.je(jit, side_exit(jit, function, state, FixnumModByZero)); asm_ccall!(asm, rb_fix_mod_fix, left, right) } @@ -2575,11 +2596,11 @@ fn gen_box_bool(asm: &mut Assembler, val: lir::Opnd) -> lir::Opnd { asm.csel_nz(Opnd::Value(Qtrue), Opnd::Value(Qfalse)) } -fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, state: &FrameState) -> lir::Opnd { +fn gen_box_fixnum(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, state: &FrameState) -> lir::Opnd { // Load the value, then test for overflow and tag it let val = asm.load_mem(val); let shifted = asm.lshift(val, Opnd::UImm(1)); - asm.jo(jit, side_exit(jit, state, BoxFixnumOverflow)); + asm.jo(jit, side_exit(jit, function, state, BoxFixnumOverflow)); asm.or(shifted, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)) } @@ -2713,33 +2734,33 @@ fn gen_has_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_typ } /// Compile a type check with a side exit -fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_type: Type, guard_type: Type, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, val_type: Type, guard_type: Type, recompile: Option, state: &FrameState) -> lir::Opnd { let is_known_heap_basic_object = val_type.is_subtype(types::HeapBasicObject); gen_incr_counter(asm, Counter::guard_type_count); if guard_type.is_subtype(types::Fixnum) { asm.test(val, Opnd::UImm(RUBY_FIXNUM_FLAG as u64)); - asm.jz(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jz(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::Flonum) { // Flonum: (val & RUBY_FLONUM_MASK) == RUBY_FLONUM_FLAG let masked = asm.and(val, Opnd::UImm(RUBY_FLONUM_MASK as u64)); asm.cmp(masked, Opnd::UImm(RUBY_FLONUM_FLAG as u64)); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::StaticSymbol) { // Static symbols have (val & 0xff) == RUBY_SYMBOL_FLAG // Use 8-bit comparison like YJIT does. // If `val` is a constant (rare but possible), put it in a register to allow masking. let val = asm.load_imm(val); asm.cmp(val.with_num_bits(8), Opnd::UImm(RUBY_SYMBOL_FLAG as u64)); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::NilClass) { asm.cmp(val, Qnil.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::TrueClass) { asm.cmp(val, Qtrue.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_subtype(types::FalseClass) { asm.cmp(val, Qfalse.into()); - asm.jne(jit, side_exit_with_recompile(jit, state, GuardType(guard_type), recompile)); + asm.jne(jit, side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile)); } else if guard_type.is_immediate() { // All immediate types' guard should have been handled above panic!("unexpected immediate guard type: {guard_type}"); @@ -2750,7 +2771,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t // TODO: Max thinks codegen should not care about the shapes of the operands except to create them. (Shopify/ruby#685) let val = asm.load_mem(val); - let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side_exit = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); if !is_known_heap_basic_object { // Check if it's a special constant asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); @@ -2767,7 +2788,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t asm.cmp(klass, Opnd::Value(expected_class)); asm.jne(jit, side_exit); } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() { - let side = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); if !is_known_heap_basic_object { // Check special constant @@ -2786,7 +2807,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t asm.cmp(tag, Opnd::UImm(builtin_type as u64)); asm.jne(jit, side); } else if guard_type.bit_equal(types::HeapBasicObject) { - let side_exit = side_exit_with_recompile(jit, state, GuardType(guard_type), recompile); + let side_exit = side_exit_with_recompile(jit, function, state, GuardType(guard_type), recompile); asm.cmp(val, Opnd::Value(Qfalse)); asm.je(jit, side_exit.clone()); asm.test(val, (RUBY_IMMEDIATE_MASK as u64).into()); @@ -2798,7 +2819,7 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, val_t } /// Compile an identity check with a side exit -fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, expected: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { if matches!(reason, SideExitReason::GuardShape(_) ) { gen_incr_counter(asm, Counter::guard_shape_count); } @@ -2809,7 +2830,7 @@ fn gen_guard_bit_equals(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, _ => panic!("gen_guard_bit_equals: unexpected hir::Const {expected:?}"), }; asm.cmp(val, expected_opnd); - asm.jnz(jit, side_exit_with_recompile(jit, state, reason, recompile)); + asm.jnz(jit, side_exit_with_recompile(jit, function, state, reason, recompile)); val } @@ -2824,18 +2845,18 @@ fn mask_to_opnd(mask: crate::hir::Const) -> Option { } /// Compile a bitmask check with a side exit if none of the masked bits are not set -fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { +fn gen_guard_any_bit_set(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, recompile: Option, state: &FrameState) -> lir::Opnd { let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_any_bit_set: unexpected hir::Const {mask:?}")); asm.test(val, mask_opnd); - asm.jz(jit, side_exit_with_recompile(jit, state, reason, recompile)); + asm.jz(jit, side_exit_with_recompile(jit, function, state, reason, recompile)); val } /// Compile a bitmask check with a side exit if any of the masked bits are set -fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { +fn gen_guard_no_bits_set(jit: &mut JITState, asm: &mut Assembler, function: &Function, val: lir::Opnd, mask: crate::hir::Const, reason: SideExitReason, state: &FrameState) -> lir::Opnd { let mask_opnd = mask_to_opnd(mask).unwrap_or_else(|| panic!("gen_guard_no_bits_set: unexpected hir::Const {mask:?}")); asm.test(val, mask_opnd); - asm.jnz(jit, side_exit(jit, state, reason)); + asm.jnz(jit, side_exit(jit, function, state, reason)); val } @@ -2930,16 +2951,22 @@ fn cfp_jit_return_for_depth(asm: &mut Assembler, depth: InlineDepth) -> Opnd { } } +fn jit_frame_next_pc(state: &FrameState) -> *const VALUE { + let opcode: usize = state.get_opcode().try_into().unwrap(); + unsafe { state.pc.offset(insn_len(opcode) as isize) } +} + +fn jit_frame_for_state(state: &FrameState, stack_map_size: usize) -> *const zjit_jit_frame { + JITFrame::new_iseq(jit_frame_next_pc(state), state.iseq, stack_map_size) +} + /// Save only the PC to CFP. Use this when you need to call gen_save_sp() /// immediately after with a custom stack size (e.g., gen_ccall_with_frame /// adjusts SP to exclude receiver and arguments). fn gen_write_jit_frame(asm: &mut Assembler, state: &FrameState, stack_map_size: usize) -> *const zjit_jit_frame { - let opcode: usize = state.get_opcode().try_into().unwrap(); - let next_pc: *const VALUE = unsafe { state.pc.offset(insn_len(opcode) as isize) }; - gen_incr_counter(asm, Counter::vm_write_jit_frame_count); asm_comment!(asm, "save JITFrame to CFP"); - let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, stack_map_size); + let jit_frame = jit_frame_for_state(state, stack_map_size); asm.mov(Opnd::mem(64, NATIVE_BASE_PTR, jit_frame_slot_offset(state.depth)), Opnd::const_ptr(jit_frame)); // CFP_PC for a live JIT frame routes through the JITFrame on the native @@ -3010,13 +3037,23 @@ fn gen_spill_locals(jit: &JITState, asm: &mut Assembler, state: &FrameState) { } /// Spill the virtual stack onto the stack. -fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // This function does not call gen_save_sp() at the moment because // gen_send_without_block_direct() spills stack slots above SP for arguments. gen_incr_counter(asm, Counter::vm_write_stack_count); asm_comment!(asm, "spill stack"); - for (idx, &insn_id) in state.stack().enumerate() { - asm.mov(Opnd::mem(64, SP, idx as i32 * SIZEOF_VALUE_I32), jit.get_opnd(insn_id)); + + let mut offset = state.stack_size() as i32; + for entry in build_stack_map(jit, function, state) { + match entry { + StackMapEntry::Opnd(opnd) => { + offset -= 1; + asm.mov(Opnd::mem(64, SP, offset * SIZEOF_VALUE_I32), opnd); + } + StackMapEntry::Skip(skip) => { + offset -= skip as i32; + } + } } } @@ -3025,38 +3062,57 @@ fn gen_spill_stack(jit: &JITState, asm: &mut Assembler, state: &FrameState) { /// Direct JIT-to-JIT calls keep cfp->sp lazy, so this must publish SP before /// writing stack slots. Otherwise spilling the stack can overwrite frame /// metadata below the real VM-stack base. -fn gen_prepare_fallback_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_prepare_fallback_call(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { gen_write_jit_frame(asm, state, 0); gen_save_sp(asm, state.stack_size()); gen_spill_locals(jit, asm, state); - gen_spill_stack(jit, asm, state); + gen_spill_stack(jit, asm, function, state); } -/// Record the Ruby stack values needed to materialize this frame after the next -/// non-leaf C call. The actual JITFrame entries are encoded by the register -/// allocator, where VReg locations on the native stack are known. -fn gen_stack_map(jit: &JITState, asm: &mut Assembler, state: &FrameState, stack_size: usize, jit_frame: *const zjit_jit_frame) { +/// Build entries for Ruby stack values that need materialization. The actual +/// JITFrame entries are encoded by the register allocator, where VReg locations +/// on the native stack are known. +fn build_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> Vec { let mut stack = Vec::new(); - for &insn_id in state.stack().take(stack_size) { - let opnd = jit.get_opnd(insn_id); - // JITFrame only supports materializing Opnd::Value or Opnd::VReg out of the frame - assert!(matches!(opnd, Opnd::Value(_) | Opnd::VReg { .. }), "FrameState should only reference Opnd::Value or Opnd::VReg, but got: {opnd:?}"); - stack.push(opnd); + let mut current_state = state.clone(); + loop { + stack.extend(current_state.stack().rev().copied().map(|insn_id| { + let opnd = jit.get_opnd(insn_id); + assert!( + matches!(opnd, Opnd::Value(_) | Opnd::VReg { .. }), + "FrameState should only reference Opnd::Value or Opnd::VReg, but got: {opnd:?}", + ); + StackMapEntry::Opnd(opnd) + })); + + let Some(caller) = current_state.caller() else { + break; + }; + stack.push(StackMapEntry::Skip(inline_frame_stack_gap(current_state.iseq))); + current_state = function.frame_state(caller); } - asm.stack_map(stack, jit_frame, state.depth); + stack +} + +fn inline_frame_stack_gap(iseq: IseqPtr) -> usize { + // The extra slot is for the callee's receiver below its local table. + // We currently never map out the stack for `invokeblock`, which doesn't + // put a receiver on cfp->sp stack. + 1 + unsafe { get_iseq_body_local_table_size(iseq) }.to_usize() + VM_ENV_DATA_SIZE.to_usize() } /// Prepare for calling a C function that may call an arbitrary method. /// Use gen_prepare_leaf_call_with_gc() if the method is leaf but allocates objects. -fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, state: &FrameState) { +fn gen_prepare_non_leaf_call(jit: &JITState, asm: &mut Assembler, function: &Function, state: &FrameState) { // TODO: Lazily materialize caller frames when needed // Save PC for backtraces and allocation tracing // and SP to avoid marking uninitialized stack slots - let jit_frame = gen_prepare_call_with_gc(asm, state, false, state.stack_size()); + let stack_map = build_stack_map(jit, function, state); + let jit_frame = gen_prepare_call_with_gc(asm, state, false, stack_map.len()); - // Spill the virtual stack in case it raises an exception + // Remember the stack map in case it raises an exception // and the interpreter uses the stack for handling the exception - gen_stack_map(jit, asm, state, state.stack_size(), jit_frame); + asm.stack_map(stack_map, jit_frame, state.depth); // Spill locals in case the method looks at caller Bindings gen_spill_locals(jit, asm, state); @@ -3136,7 +3192,7 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C } /// Stack overflow check: fails if CFP<=SP at any point in the callee. -fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &FrameState, stack_growth: usize) { +fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, function: &Function, state: &FrameState, stack_growth: usize) { asm_comment!(asm, "stack overflow check"); // vm_push_frame() checks it against a decremented cfp, and CHECK_VM_STACK_OVERFLOW0 // adds to the margin another control frame with `&bounds[1]`. @@ -3145,7 +3201,7 @@ fn gen_stack_overflow_check(jit: &mut JITState, asm: &mut Assembler, state: &Fra let peak_offset = (cfp_growth + stack_growth) * SIZEOF_VALUE; let stack_limit = asm.lea(Opnd::mem(64, SP, peak_offset as i32)); asm.cmp(CFP, stack_limit); - asm.jbe(jit, side_exit(jit, state, StackOverflow)); + asm.jbe(jit, side_exit(jit, function, state, StackOverflow)); } /// Convert ISEQ into High-level IR @@ -3179,14 +3235,14 @@ fn compile_iseq(iseq: IseqPtr) -> Result { } /// Build a Target::SideExit -fn side_exit(jit: &JITState, state: &FrameState, reason: SideExitReason) -> Target { - let exit = build_side_exit(jit, state); +fn side_exit(jit: &JITState, function: &Function, state: &FrameState, reason: SideExitReason) -> Target { + let exit = build_side_exit(jit, function, state); Target::SideExit(Box::new(SideExitTarget { exit, reason })) } /// Build a Target::SideExit that optionally triggers exit_recompile on the exit path. -fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExitReason, recompile: Option) -> Target { - let mut exit = build_side_exit(jit, state); +fn side_exit_with_recompile(jit: &JITState, function: &Function, state: &FrameState, reason: SideExitReason, recompile: Option) -> Target { + let mut exit = build_side_exit(jit, function, state); exit.recompile = recompile.map(|_| SideExitRecompile { compiled_iseq: Opnd::Value(VALUE::from(jit.iseq())), insn_idx: state.insn_idx() as u32, @@ -3195,7 +3251,7 @@ fn side_exit_with_recompile(jit: &JITState, state: &FrameState, reason: SideExit } /// Build a side-exit context -fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { +fn build_side_exit(jit: &JITState, function: &Function, state: &FrameState) -> SideExit { let mut stack = Vec::new(); for &insn_id in state.stack() { stack.push(jit.get_opnd(insn_id)); @@ -3211,10 +3267,23 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { stack, locals, iseq: state.iseq, + stack_map: build_caller_stack_map(jit, function, state), recompile: None, } } +fn build_caller_stack_map(jit: &JITState, function: &Function, state: &FrameState) -> Option { + let caller = state.caller()?; + let caller_state = function.frame_state(caller); + let stack_map = build_stack_map(jit, function, &caller_state); + if stack_map.is_empty() { + return None; + } + + let jit_frame = jit_frame_for_state(&caller_state, stack_map.len()); + Some(StackMap::new(stack_map, jit_frame, caller_state.depth)) +} + #[cfg(target_arch = "x86_64")] macro_rules! c_callable { ($(#[$outer:meta])* @@ -3632,15 +3701,15 @@ fn gen_push_opnds(jit: &JITState, asm: &mut Assembler, opnds: &[Opnd]) -> lir::O argv } -fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, opt: usize, values: Vec, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_toregexp(jit: &mut JITState, asm: &mut Assembler, function: &Function, opt: usize, values: Vec, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); let first_opnd_ptr = gen_push_opnds(jit, asm, &values); asm_ccall!(asm, rb_reg_new_from_values, values.len().into(), first_opnd_ptr, opt.into()) } -fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, strings: Vec, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_concat(jit: &mut JITState, asm: &mut Assembler, function: &Function, strings: Vec, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); let first_string_ptr = gen_push_opnds(jit, asm, &strings); asm_ccall!(asm, rb_str_concat_literals, strings.len().into(), first_string_ptr) @@ -3681,13 +3750,13 @@ fn gen_string_setbyte_fixnum(asm: &mut Assembler, string: Opnd, index: Opnd, val asm_ccall!(asm, rb_str_setbyte, string, index, value) } -fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_append(jit: &mut JITState, asm: &mut Assembler, function: &Function, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_str_buf_append, string, val) } -fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { - gen_prepare_non_leaf_call(jit, asm, state); +fn gen_string_append_codepoint(jit: &mut JITState, asm: &mut Assembler, function: &Function, string: Opnd, val: Opnd, state: &FrameState) -> Opnd { + gen_prepare_non_leaf_call(jit, asm, function, state); asm_ccall!(asm, rb_jit_str_concat_codepoint, string, val) } diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index efd158bfe3eeb4..51055badf29cad 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -6275,6 +6275,23 @@ fn test_inlined_method_with_invokeblock() { }); } +#[test] +fn test_inlined_method_with_invokeblock_raise_materializes_stack() { + with_inlining(|| { + assert_snapshot!(assert_inlines_allowing_exits(" + def callee = [1, 2, yield] + def test + callee { raise } + rescue + :rescued + end + + test + test + "), @":rescued"); + }); +} + #[test] fn test_inlined_method_with_block_param() { with_inlining(|| { @@ -6293,6 +6310,24 @@ fn test_inlined_method_with_block_param() { }); } +#[test] +fn test_inlined_method_that_forwards_block_arg_raise_materializes_stack() { + with_inlining(|| { + assert_snapshot!(assert_inlines_allowing_exits(" + def inner = yield + def callee(&block) = [1, 2, inner(&block)] + def test + callee { raise } + rescue + :rescued + end + + test + test + "), @":rescued"); + }); +} + #[test] fn test_inlined_method_that_forwards_block_arg() { // The callee captures a literal block in `&block` and forwards it on to diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index a43038e665ce2b..3a76b2a16f10e7 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -235,6 +235,7 @@ pub const VM_ENV_DATA_INDEX_FLAGS: u32 = 0; pub const VM_BLOCK_HANDLER_NONE: u32 = 0; pub const SHAPE_ID_NUM_BITS: u32 = 32; pub const ZJIT_STACK_MAP_VREG_TAG: u32 = 8; +pub const ZJIT_STACK_MAP_SKIP_TAG: u32 = 16; pub const ZJIT_STACK_MAP_SHIFT: u32 = 8; pub const ZJIT_JIT_RETURN_C_FRAME: u32 = 1; pub const RB_GC_ZJIT_FASTPATH_DATA_WORDS: u32 = 19; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 4d3aea54baca7c..028196d7d4ff59 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -4862,9 +4862,12 @@ impl Function { // callee sits one level deeper (caller_depth + 1). let caller_depth = self.frame_depth(state); + // The callee's perspective of the stack is with the receiver and arguments popped off. + let caller_stack_size = call_state.stack_size() - args.len() - 1; // -1 for receiver + let post_send_caller = self.new_insn(Insn::Snapshot { state: Box::new(call_state.with_stack_size(caller_stack_size)) }); let mode = AddIseqMode::Inlined { return_block: continuation, - caller: state, + caller: post_send_caller, depth: caller_depth + 1, jit_entry_idx: passed_opt_num, }; @@ -4992,6 +4995,10 @@ impl Function { // The original SendDirect result is now the continuation's return value param. self.make_equal_to(send_insn_id, return_val_param); + // Keep the caller FrameState that inlined callee Snapshots point at + // as a distinct Snapshot instead of rewriting the consumed SendDirect. + self.push_insn_id(block, post_send_caller); + // Insert PushLightweightFrame and jump to callee body entry. self.push_insn(block, Insn::PushInlineFrame { iseq, cme, recv, args: args.clone(), blockiseq, state, @@ -6847,7 +6854,7 @@ pub struct FrameState { stack: Vec, locals: Vec, - /// `InsnId` of the caller's call-site `Snapshot` for inlined frames; `None` + /// `InsnId` of the caller's post-send `Snapshot` for inlined frames; `None` /// for non-inlined frames. Stored as an instruction reference rather than /// an owned `FrameState` so that value remapping in the caller's `Snapshot` /// propagates here automatically, and so the caller's state has a single @@ -6882,6 +6889,13 @@ impl FrameState { state } + /// Return itself with a truncated stack. + pub fn with_stack_size(&self, stack_size: usize) -> Self { + let mut state = self.clone(); + state.stack.truncate(stack_size); + state + } + /// Return itself with send args replaced. Used when kwargs are reordered/synthesized for callee. /// `original_argc` is the number of args originally on the stack (before processing). fn with_replaced_args(&self, new_args: &[InsnId], original_argc: usize) -> Self { @@ -6918,7 +6932,7 @@ impl FrameState { } /// Construct a `FrameState` for an inlined callee. `caller` is the `InsnId` - /// of the caller's call-site `Snapshot`; `depth` is this frame's inlining depth. + /// of the caller's post-send `Snapshot`; `depth` is this frame's inlining depth. fn inlined(iseq: IseqPtr, caller: InsnId, depth: InlineDepth) -> FrameState { FrameState { caller: Some(caller), depth, ..FrameState::new(iseq) } } @@ -6933,6 +6947,10 @@ impl FrameState { self.stack.iter() } + pub fn caller(&self) -> Option { + self.caller + } + /// Iterate over all local variables pub fn locals(&self) -> Iter<'_, InsnId> { self.locals.iter() @@ -7244,7 +7262,7 @@ enum AddIseqMode { Standalone, Inlined { return_block: BlockId, - /// The caller's call-site `Snapshot`. Allows side-exits to restore the outer frame. + /// The caller's post-send `Snapshot`. Allows side-exits to restore the outer frame. caller: InsnId, /// Inlining depth of every frame emitted for the callee. depth: InlineDepth, @@ -7315,7 +7333,7 @@ fn add_iseq_to_hir( let mut profiles = ProfileOracle::new(); // Build the initial FrameState for a block being translated. In inlined - // mode it carries the caller's call-site Snapshot and this frame's depth; + // mode it carries the caller's post-send Snapshot and this frame's depth; // because every Snapshot emitted for the callee is cloned from one of these // initial states, those values propagate to the whole inlined body without a // separate rewrite pass. diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index b82417d44aa7e7..595efc4e66b2a8 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -1546,11 +1546,11 @@ mod hir_opt_tests { v11:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v32:NilClass = Const Value(nil) + v33:NilClass = Const Value(nil) PushInlineFrame v20 (0x1038), v11 CheckInterrupts PopInlineFrame - Return v32 + Return v33 "); } @@ -1708,10 +1708,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, Integer@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:BasicObject = InvokeBuiltin rb_f_integer1, v20, v11 + v28:BasicObject = InvokeBuiltin rb_f_integer1, v20, v11 CheckInterrupts PopInlineFrame - Return v27 + Return v28 "); } @@ -1740,10 +1740,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 - v30:ArrayExact = NewArray + v31:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v30 + Return v31 "); } @@ -1768,13 +1768,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v18 (0x1038) - v25:Fixnum[1] = Const Value(1) - v33:Fixnum[2] = Const Value(2) + v26:Fixnum[1] = Const Value(1) + v34:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v60:Fixnum[3] = Const Value(3) + v61:Fixnum[3] = Const Value(3) CheckInterrupts PopInlineFrame - Return v60 + Return v61 "); } @@ -1800,12 +1800,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:Fixnum[2] = Const Value(2) + v28:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v53:Fixnum[5] = Const Value(5) + v54:Fixnum[5] = Const Value(5) CheckInterrupts PopInlineFrame - Return v53 + Return v54 "); } @@ -1833,10 +1833,10 @@ mod hir_opt_tests { v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v46:Fixnum[7] = Const Value(7) + v47:Fixnum[7] = Const Value(7) CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -1862,19 +1862,19 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, target@0x1008, cme:0x1010) v44:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v44 (0x1038) - v56:Fixnum[1] = Const Value(1) - v66:Fixnum[2] = Const Value(2) - v76:Fixnum[3] = Const Value(3) - v86:Fixnum[4] = Const Value(4) - v101:ArrayExact = NewArray v56, v66, v76, v86 + v57:Fixnum[1] = Const Value(1) + v67:Fixnum[2] = Const Value(2) + v77:Fixnum[3] = Const Value(3) + v87:Fixnum[4] = Const Value(4) + v102:ArrayExact = NewArray v57, v67, v77, v87 CheckInterrupts PopInlineFrame v14:Fixnum[10] = Const Value(10) v16:Fixnum[20] = Const Value(20) v18:Fixnum[30] = Const Value(30) PushInlineFrame v44 (0x1038), v14, v16, v18 - v121:Fixnum[4] = Const Value(4) - v136:ArrayExact = NewArray v14, v16, v18, v121 + v123:Fixnum[4] = Const Value(4) + v138:ArrayExact = NewArray v14, v16, v18, v123 PopInlineFrame v24:Fixnum[10] = Const Value(10) v26:Fixnum[20] = Const Value(20) @@ -1882,7 +1882,7 @@ mod hir_opt_tests { v30:Fixnum[40] = Const Value(40) v32:Fixnum[50] = Const Value(50) v34:BasicObject = Send v44, :target, v24, v26, v28, v30, v32 # SendFallbackReason: Argument count does not match parameter count - v37:ArrayExact = NewArray v101, v136, v34 + v37:ArrayExact = NewArray v102, v138, v34 CheckInterrupts Return v37 "); @@ -3797,10 +3797,10 @@ mod hir_opt_tests { PatchPoint MethodRedefined(C@0x1008, foo@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact:C] = GuardType v10, ObjectSubclass[class_exact:C] recompile PushInlineFrame v23 (0x1040) - v29:ArrayExact = NewArray + v30:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v29 + Return v30 "); } @@ -3828,25 +3828,25 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v51:NilClass = Const Value(nil) + v52:NilClass = Const Value(nil) PushInlineFrame v22 (0x1038), v11, v13 - v33:CPtr = GetEP 0 - v34:CUInt64 = LoadField v33, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v35:CBool = IsBlockParamModified v34 - CondBranch v35, bb6(), bb7() + v34:CPtr = GetEP 0 + v35:CUInt64 = LoadField v34, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v36:CBool = IsBlockParamModified v35 + CondBranch v36, bb6(), bb7() bb6(): - v37:BasicObject = LoadField v33, :block@0x1041 - Jump bb8(v37, v37) + v38:BasicObject = LoadField v34, :block@0x1041 + Jump bb8(v38, v38) bb7(): - v39:CInt64 = LoadField v33, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v40:CInt64 = GuardAnyBitSet v39, CUInt64(1) recompile - v41:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) - Jump bb8(v41, v51) - bb8(v31:BasicObject, v32:BasicObject): - v46:BasicObject = Send v31, :call, v11, v13 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall + v40:CInt64 = LoadField v34, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v41:CInt64 = GuardAnyBitSet v40, CUInt64(1) recompile + v42:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) + Jump bb8(v42, v52) + bb8(v32:BasicObject, v33:BasicObject): + v47:BasicObject = Send v32, :call, v11, v13 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -3986,7 +3986,7 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v20 (0x1038), v11 - v27:Fixnum[80] = Const Value(80) + v28:Fixnum[80] = Const Value(80) CheckInterrupts PopInlineFrame Return v11 @@ -4018,12 +4018,12 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v22 (0x1038), v11, v13 - v30:Fixnum[80] = Const Value(80) + v31:Fixnum[80] = Const Value(80) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v66:Fixnum[110] = Const Value(110) + v67:Fixnum[110] = Const Value(110) CheckInterrupts PopInlineFrame - Return v66 + Return v67 "); } @@ -4050,12 +4050,12 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1038), v11, v13 - v33:ArrayExact = NewArray v11, v13 + v34:ArrayExact = NewArray v11, v13 CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); } @@ -4083,12 +4083,12 @@ mod hir_opt_tests { v15:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1038), v13, v15, v11 - v38:ArrayExact = NewArray v13, v15, v11 + v39:ArrayExact = NewArray v13, v15, v11 CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -4116,12 +4116,12 @@ mod hir_opt_tests { v15:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1038), v11, v15, v13 - v38:ArrayExact = NewArray v11, v15, v13 + v39:ArrayExact = NewArray v11, v15, v13 CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -4148,12 +4148,12 @@ mod hir_opt_tests { v13:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1038), v11, v13 - v33:ArrayExact = NewArray v11, v13 + v34:ArrayExact = NewArray v11, v13 CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); } @@ -4181,21 +4181,21 @@ mod hir_opt_tests { v15:Fixnum[4] = Const Value(4) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v71:Fixnum[0] = Const Value(0) + v72:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v11, v13, v15 - v51:Fixnum[2] = Const Value(2) - v65:ArrayExact = NewArray v51, v13 + v52:Fixnum[2] = Const Value(2) + v66:ArrayExact = NewArray v52, v13 CheckInterrupts PopInlineFrame v20:Fixnum[1] = Const Value(1) v22:Fixnum[2] = Const Value(2) v24:Fixnum[4] = Const Value(4) v26:Fixnum[3] = Const Value(3) - v92:Fixnum[0] = Const Value(0) + v94:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v20, v22, v26, v24 - v87:ArrayExact = NewArray v22, v26 + v89:ArrayExact = NewArray v22, v26 PopInlineFrame - v30:ArrayExact = NewArray v65, v87 + v30:ArrayExact = NewArray v66, v89 Return v30 "); } @@ -4224,21 +4224,21 @@ mod hir_opt_tests { v34:Fixnum[4] = Const Value(4) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v37:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v73:Fixnum[0] = Const Value(0) + v74:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v11, v13, v34 - v51:Fixnum[2] = Const Value(2) - v67:ArrayExact = NewArray v11, v51, v13, v34 + v52:Fixnum[2] = Const Value(2) + v68:ArrayExact = NewArray v11, v52, v13, v34 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) v20:Fixnum[2] = Const Value(2) v22:Fixnum[40] = Const Value(40) v24:Fixnum[30] = Const Value(30) - v96:Fixnum[0] = Const Value(0) + v98:Fixnum[0] = Const Value(0) PushInlineFrame v37 (0x1038), v18, v20, v24, v22 - v91:ArrayExact = NewArray v18, v20, v24, v22 + v93:ArrayExact = NewArray v18, v20, v24, v22 PopInlineFrame - v28:ArrayExact = NewArray v67, v91 + v28:ArrayExact = NewArray v68, v93 Return v28 "); } @@ -4307,7 +4307,7 @@ mod hir_opt_tests { v11:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v32:Fixnum[0] = Const Value(0) + v33:Fixnum[0] = Const Value(0) PushInlineFrame v20 (0x1038), v11 CheckInterrupts PopInlineFrame @@ -4424,14 +4424,14 @@ mod hir_opt_tests { v17:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v20:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v37:Fixnum[0] = Const Value(0) + v38:Fixnum[0] = Const Value(0) PushInlineFrame v20 (0x1038), v17 - v29:Fixnum[1] = Const Value(1) + v30:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1040, +@0x1048, cme:0x1050) - v46:Fixnum[2] = Const Value(2) + v47:Fixnum[2] = Const Value(2) CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -4729,12 +4729,12 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(C@0x1008) PatchPoint MethodRedefined(C@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v48 (0x1068), v17 - v65:CShape = LoadField v48, :shape_id@0x1070 - v66:CShape[0x1071] = GuardBitEquals v65, CShape(0x1071) recompile + v66:CShape = LoadField v48, :shape_id@0x1070 + v67:CShape[0x1071] = GuardBitEquals v66, CShape(0x1071) recompile StoreField v48, :@x@0x1072, v17 WriteBarrier v48, v17 - v69:CShape[0x1073] = Const CShape(0x1073) - StoreField v48, :shape_id@0x1070, v69 + v70:CShape[0x1073] = Const CShape(0x1073) + StoreField v48, :shape_id@0x1070, v70 CheckInterrupts PopInlineFrame Return v48 @@ -4829,22 +4829,22 @@ mod hir_opt_tests { v46:Fixnum[0] = Const Value(0) PatchPoint NoSingletonClass(Hash@0x1008) PatchPoint MethodRedefined(Hash@0x1008, initialize@0x1038, cme:0x1040) - v97:Fixnum[0] = Const Value(0) - v98:NilClass = Const Value(nil) + v98:Fixnum[0] = Const Value(0) + v99:NilClass = Const Value(nil) PushInlineFrame v45 (0x1068), v46 - v64:TrueClass = Const Value(true) - v82:CPtr = GetEP 0 - v83:CUInt64 = LoadField v82, :VM_ENV_DATA_INDEX_FLAGS@0x1070 - v84:CBool = IsBlockParamModified v83 - CondBranch v84, bb11(), bb12() + v65:TrueClass = Const Value(true) + v83:CPtr = GetEP 0 + v84:CUInt64 = LoadField v83, :VM_ENV_DATA_INDEX_FLAGS@0x1070 + v85:CBool = IsBlockParamModified v84 + CondBranch v85, bb11(), bb12() bb11(): - v86:BasicObject = LoadField v82, :block@0x1071 - Jump bb13(v86) + v87:BasicObject = LoadField v83, :block@0x1071 + Jump bb13(v87) bb12(): - v88:BasicObject = GetBlockParam :block, l0, EP@4 - Jump bb13(v88) - bb13(v81:BasicObject): - v91:BasicObject = InvokeBuiltin rb_hash_init, v45, v46, v64, v64, v81 + v89:BasicObject = GetBlockParam :block, l0, EP@4 + Jump bb13(v89) + bb13(v82:BasicObject): + v92:BasicObject = InvokeBuiltin rb_hash_init, v45, v46, v65, v65, v82 CheckInterrupts PopInlineFrame Return v45 @@ -7334,10 +7334,10 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, []@0x1010, cme:0x1018) PushInlineFrame v11 (0x1040), v13 - v30:ArrayExact = NewArray + v31:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v30 + Return v31 "); } @@ -7396,10 +7396,10 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Array@0x1008) PatchPoint MethodRedefined(Array@0x1008, max@0x1010, cme:0x1018) PushInlineFrame v11 (0x1040) - v26:ArrayExact = NewArray + v27:ArrayExact = NewArray CheckInterrupts PopInlineFrame - Return v26 + Return v27 "); } @@ -9622,11 +9622,11 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile PushInlineFrame v18 (0x1038) - v24:Fixnum[1] = Const Value(1) - v26:BasicObject = InvokeBlock v24 # SendFallbackReason: InvokeBlock: not yet specialized + v25:Fixnum[1] = Const Value(1) + v27:BasicObject = InvokeBlock v25 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts PopInlineFrame - Return v26 + Return v27 "); } @@ -9655,49 +9655,49 @@ mod hir_opt_tests { bb3(v6:BasicObject): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v18:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v71:NilClass = Const Value(nil) + v72:NilClass = Const Value(nil) PushInlineFrame v18 (0x1038) - v27:CPtr = GetEP 0 - v28:CUInt64 = LoadField v27, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v29:CBool = IsBlockParamModified v28 - CondBranch v29, bb7(), bb8() + v28:CPtr = GetEP 0 + v29:CUInt64 = LoadField v28, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v30:CBool = IsBlockParamModified v29 + CondBranch v30, bb7(), bb8() bb7(): - v31:BasicObject = LoadField v27, :blk@0x1041 - Jump bb9(v31, v31) + v32:BasicObject = LoadField v28, :blk@0x1041 + Jump bb9(v32, v32) bb8(): - v33:CInt64 = LoadField v27, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v34:CInt64[0] = GuardBitEquals v33, CInt64(0) recompile - v35:NilClass = Const Value(nil) - Jump bb9(v35, v71) - bb9(v25:BasicObject, v26:BasicObject): + v34:CInt64 = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v35:CInt64[0] = GuardBitEquals v34, CInt64(0) recompile + v36:NilClass = Const Value(nil) + Jump bb9(v36, v72) + bb9(v26:BasicObject, v27:BasicObject): CheckInterrupts - v39:CBool = Test v25 - CondBranch v39, bb10(), bb6(v18, v26) + v40:CBool = Test v26 + CondBranch v40, bb10(), bb6(v18, v27) bb10(): - v46:CPtr = GetEP 0 - v47:CUInt64 = LoadField v46, :VM_ENV_DATA_INDEX_FLAGS@0x1040 - v48:CBool = IsBlockParamModified v47 - CondBranch v48, bb11(), bb12() + v47:CPtr = GetEP 0 + v48:CUInt64 = LoadField v47, :VM_ENV_DATA_INDEX_FLAGS@0x1040 + v49:CBool = IsBlockParamModified v48 + CondBranch v49, bb11(), bb12() bb11(): - v50:BasicObject = LoadField v46, :blk@0x1041 - Jump bb13(v50, v50) + v51:BasicObject = LoadField v47, :blk@0x1041 + Jump bb13(v51, v51) bb12(): - v52:CInt64 = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 - v53:CInt64 = GuardAnyBitSet v52, CUInt64(1) recompile - v54:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) - Jump bb13(v54, v26) - bb13(v44:BasicObject, v45:BasicObject): - v57:BasicObject = Send v44, :call # SendFallbackReason: SendWithoutBlock: no profile data available - CheckInterrupts - Jump bb4(v57) - bb6(v62:ObjectSubclass[class_exact*:Object@VALUE(0x1000)], v63:BasicObject): - v66:Fixnum[42] = Const Value(42) - CheckInterrupts - Jump bb4(v66) - bb4(v72:BasicObject): + v53:CInt64 = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1042 + v54:CInt64 = GuardAnyBitSet v53, CUInt64(1) recompile + v55:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1048)) + Jump bb13(v55, v27) + bb13(v45:BasicObject, v46:BasicObject): + v58:BasicObject = Send v45, :call # SendFallbackReason: SendWithoutBlock: no profile data available + CheckInterrupts + Jump bb4(v58) + bb6(v63:ObjectSubclass[class_exact*:Object@VALUE(0x1000)], v64:BasicObject): + v67:Fixnum[42] = Const Value(42) + CheckInterrupts + Jump bb4(v67) + bb4(v73:BasicObject): PopInlineFrame CheckInterrupts - Return v72 + Return v73 "); } @@ -9726,12 +9726,12 @@ mod hir_opt_tests { PatchPoint SingleRactorMode PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) v19:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - v48:NilClass = Const Value(nil) + v49:NilClass = Const Value(nil) PushInlineFrame v19 (0x1038) CheckInterrupts - v43:Fixnum[42] = Const Value(42) + v44:Fixnum[42] = Const Value(42) PopInlineFrame - Return v43 + Return v44 "); } @@ -15508,11 +15508,11 @@ mod hir_opt_tests { v21:RubyValue = LoadField v18, :VM_ENV_DATA_INDEX_SPECVAL@0x1048 v22:FalseClass = GuardBitEquals v21, Value(false) PushInlineFrame v6 (0x1050) - v28:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) - v29:StringExact = StringCopy v28 + v29:StringExact[VALUE(0x1058)] = Const Value(VALUE(0x1058)) + v30:StringExact = StringCopy v29 CheckInterrupts PopInlineFrame - Return v29 + Return v30 "); } @@ -15582,15 +15582,15 @@ mod hir_opt_tests { v31:RubyValue = LoadField v28, :VM_ENV_DATA_INDEX_SPECVAL@0x1050 v32:FalseClass = GuardBitEquals v31, Value(false) PushInlineFrame v9 (0x1058), v10 - v44:Fixnum[2] = Const Value(2) + v45:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1060, *@0x1068, cme:0x1070) - v58:Fixnum = GuardType v10, Fixnum recompile - v59:Fixnum = FixnumMult v58, v44 + v59:Fixnum = GuardType v10, Fixnum recompile + v60:Fixnum = FixnumMult v59, v45 CheckInterrupts PopInlineFrame v18:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1060, +@0x1098, cme:0x10a0) - v37:Fixnum = FixnumAdd v59, v18 + v37:Fixnum = FixnumAdd v60, v18 Return v37 "); } @@ -16938,10 +16938,10 @@ mod hir_opt_tests { v43:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v43 (0x1040), v23 PatchPoint MethodRedefined(Integer@0x1048, to_s@0x1050, cme:0x1058) - v63:StringExact = CCallVariadic v23, :Integer#to_s@0x1080 + v64:StringExact = CCallVariadic v23, :Integer#to_s@0x1080 CheckInterrupts PopInlineFrame - Return v63 + Return v64 bb4(v30:BasicObject, v31:Falsy): v35:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) v36:StringExact = StringCopy v35 @@ -18051,11 +18051,11 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v44:Fixnum = GuardType v10, Fixnum recompile - v46:Fixnum = FixnumAdd v44, v44 + v45:Fixnum = GuardType v10, Fixnum recompile + v47:Fixnum = FixnumAdd v45, v45 CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -18092,24 +18092,24 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, clamp_nonneg@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[0] = Const Value(0) + v32:Fixnum[0] = Const Value(0) PatchPoint MethodRedefined(Integer@0x1048, <@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:BoolExact = FixnumLt v62, v31 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:BoolExact = FixnumLt v63, v32 CheckInterrupts - v37:CBool = Test v63 - CondBranch v37, bb7(), bb6(v23, v62) + v38:CBool = Test v64 + CondBranch v38, bb7(), bb6(v23, v63) bb7(): - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) CheckInterrupts - Jump bb4(v42) - bb6(v47:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v48:Fixnum): + Jump bb4(v43) + bb6(v48:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v49:Fixnum): CheckInterrupts - Jump bb4(v48) - bb4(v56:Fixnum): + Jump bb4(v49) + bb4(v57:Fixnum): PopInlineFrame CheckInterrupts - Return v56 + Return v57 "); } @@ -18142,13 +18142,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_one@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[1] = Const Value(1) + v32:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v45:Fixnum = GuardType v10, Fixnum recompile - v46:Fixnum = FixnumAdd v45, v31 + v46:Fixnum = GuardType v10, Fixnum recompile + v47:Fixnum = FixnumAdd v46, v32 CheckInterrupts PopInlineFrame - Return v46 + Return v47 "); } @@ -18205,10 +18205,10 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 PatchPoint MethodRedefined(Object@0x1008, inner@0x1048, cme:0x1050) - v43:BasicObject = SendDirect v23, 0x1078, :inner (0x1088), v10 + v44:BasicObject = SendDirect v23, 0x1078, :inner (0x1088), v10 CheckInterrupts PopInlineFrame - Return v43 + Return v44 "); } @@ -18306,15 +18306,15 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[10] = Const Value(10) - v40:Fixnum[100] = Const Value(100) + v32:Fixnum[10] = Const Value(10) + v41:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v70:Fixnum = GuardType v10, Fixnum recompile - v71:Fixnum = FixnumAdd v70, v31 - v75:Fixnum = FixnumAdd v71, v40 + v71:Fixnum = GuardType v10, Fixnum recompile + v72:Fixnum = FixnumAdd v71, v32 + v76:Fixnum = FixnumAdd v72, v41 CheckInterrupts PopInlineFrame - Return v75 + Return v76 "); } @@ -18363,14 +18363,14 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opts@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10, v16 - v33:Fixnum[100] = Const Value(100) + v34:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:Fixnum = FixnumAdd v62, v16 - v67:Fixnum = FixnumAdd v63, v33 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:Fixnum = FixnumAdd v63, v16 + v68:Fixnum = FixnumAdd v64, v34 CheckInterrupts PopInlineFrame - Return v67 + Return v68 "); } @@ -18419,13 +18419,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, maybe_rescue@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v31:Fixnum[1] = Const Value(1) + v32:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v46:Fixnum = GuardType v10, Fixnum recompile - v47:Fixnum = FixnumAdd v46, v31 + v47:Fixnum = GuardType v10, Fixnum recompile + v48:Fixnum = FixnumAdd v47, v32 CheckInterrupts PopInlineFrame - Return v47 + Return v48 "); } @@ -18518,24 +18518,24 @@ mod hir_opt_tests { v23:ObjectSubclass[class_exact:Child] = GuardType v10, ObjectSubclass[class_exact:Child] recompile PushInlineFrame v23 (0x1040) PatchPoint MethodRedefined(Parent@0x1048, greet@0x1010, cme:0x1050) - v46:CPtr = GetEP 0 - v47:RubyValue = LoadField v46, :VM_ENV_DATA_INDEX_ME_CREF@0x1078 - v48:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v47, Value(VALUE(0x1018)) - v49:RubyValue = LoadField v46, :VM_ENV_DATA_INDEX_SPECVAL@0x1079 - v50:FalseClass = GuardBitEquals v49, Value(false) + v47:CPtr = GetEP 0 + v48:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_ME_CREF@0x1078 + v49:CallableMethodEntry[VALUE(0x1018)] = GuardBitEquals v48, Value(VALUE(0x1018)) + v50:RubyValue = LoadField v47, :VM_ENV_DATA_INDEX_SPECVAL@0x1079 + v51:FalseClass = GuardBitEquals v50, Value(false) PushInlineFrame v23 (0x1040) - v61:StringExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) - v62:StringExact = StringCopy v61 + v63:StringExact[VALUE(0x1080)] = Const Value(VALUE(0x1080)) + v64:StringExact = StringCopy v63 CheckInterrupts PopInlineFrame - v32:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) - v33:StringExact = StringCopy v32 + v33:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) + v34:StringExact = StringCopy v33 PatchPoint NoSingletonClass(String@0x1090) PatchPoint MethodRedefined(String@0x1090, +@0x1098, cme:0x10a0) - v56:BasicObject = CCallWithFrame v62, :String#+@0x10c8, v33 + v57:BasicObject = CCallWithFrame v64, :String#+@0x10c8, v34 CheckInterrupts PopInlineFrame - Return v56 + Return v57 "); } @@ -18585,12 +18585,12 @@ mod hir_opt_tests { v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v27 (0x1040), v10, v16, v18 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v54:Fixnum = GuardType v10, Fixnum recompile - v55:Fixnum = FixnumAdd v54, v16 - v59:Fixnum = FixnumAdd v55, v18 + v55:Fixnum = GuardType v10, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v16 + v60:Fixnum = FixnumAdd v56, v18 CheckInterrupts PopInlineFrame - Return v59 + Return v60 "); } @@ -18639,13 +18639,13 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_opt_post@0x1010, cme:0x1018) v23:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v23 (0x1040), v10 - v30:Fixnum[10] = Const Value(10) + v31:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v55:Fixnum = GuardType v10, Fixnum - v56:Fixnum = FixnumAdd v30, v55 + v56:Fixnum = GuardType v10, Fixnum + v57:Fixnum = FixnumAdd v31, v56 CheckInterrupts PopInlineFrame - Return v56 + Return v57 "); } @@ -18695,14 +18695,14 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, add_lead_opt_post@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10, v16 - v33:Fixnum[10] = Const Value(10) + v34:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v62:Fixnum = GuardType v10, Fixnum recompile - v63:Fixnum = FixnumAdd v62, v33 - v67:Fixnum = FixnumAdd v63, v16 + v63:Fixnum = GuardType v10, Fixnum recompile + v64:Fixnum = FixnumAdd v63, v34 + v68:Fixnum = FixnumAdd v64, v16 CheckInterrupts PopInlineFrame - Return v67 + Return v68 "); } @@ -18747,14 +18747,14 @@ mod hir_opt_tests { v16:Fixnum[5] = Const Value(5) PatchPoint MethodRedefined(Object@0x1008, add_kw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v16 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v16 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18799,14 +18799,14 @@ mod hir_opt_tests { v16:Fixnum[50] = Const Value(50) PatchPoint MethodRedefined(Object@0x1008, add_optkw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v16 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v16 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v16 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18851,14 +18851,14 @@ mod hir_opt_tests { v22:Fixnum[10] = Const Value(10) PatchPoint MethodRedefined(Object@0x1008, add_optkw@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v42:Fixnum[0] = Const Value(0) + v43:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1040), v10, v22 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v49:Fixnum = GuardType v10, Fixnum recompile - v50:Fixnum = FixnumAdd v49, v22 + v50:Fixnum = GuardType v10, Fixnum recompile + v51:Fixnum = FixnumAdd v50, v22 CheckInterrupts PopInlineFrame - Return v50 + Return v51 "); } @@ -18905,19 +18905,19 @@ mod hir_opt_tests { v18:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Object@0x1008, add_kws@0x1010, cme:0x1018) v28:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v60:Fixnum[0] = Const Value(0) + v61:Fixnum[0] = Const Value(0) PushInlineFrame v28 (0x1040), v10, v18, v16 - v39:Fixnum[100] = Const Value(100) + v40:Fixnum[100] = Const Value(100) PatchPoint MethodRedefined(Integer@0x1048, *@0x1050, cme:0x1058) - v67:Fixnum = GuardType v10, Fixnum recompile - v68:Fixnum = FixnumMult v67, v39 - v81:Fixnum[20] = Const Value(20) + v68:Fixnum = GuardType v10, Fixnum recompile + v69:Fixnum = FixnumMult v68, v40 + v82:Fixnum[20] = Const Value(20) PatchPoint MethodRedefined(Integer@0x1048, +@0x1080, cme:0x1088) - v76:Fixnum = FixnumAdd v68, v81 - v80:Fixnum = FixnumAdd v76, v16 + v77:Fixnum = FixnumAdd v69, v82 + v81:Fixnum = FixnumAdd v77, v16 CheckInterrupts PopInlineFrame - Return v80 + Return v81 "); } @@ -18963,26 +18963,26 @@ mod hir_opt_tests { v22:NilClass = Const Value(nil) PatchPoint MethodRedefined(Object@0x1008, add_optkw_dyn@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v63:Fixnum[1] = Const Value(1) + v64:Fixnum[1] = Const Value(1) PushInlineFrame v25 (0x1040), v10, v22 - v33:BoolExact = FixnumBitCheck v63, 0 + v34:BoolExact = FixnumBitCheck v64, 0 CheckInterrupts - v36:CBool = Test v33 - CondBranch v36, bb6(v25, v10, v22, v63), bb7() + v37:CBool = Test v34 + CondBranch v37, bb6(v25, v10, v22, v64), bb7() bb7(): - v42:Fixnum[2] = Const Value(2) + v43:Fixnum[2] = Const Value(2) PatchPoint MethodRedefined(Integer@0x1048, *@0x1050, cme:0x1058) - v70:Fixnum = GuardType v10, Fixnum recompile - v71:Fixnum = FixnumMult v70, v42 - Jump bb6(v25, v70, v71, v63) - bb6(v48:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v49:BasicObject, v50:NilClass|Fixnum, v51:Fixnum[1]): + v71:Fixnum = GuardType v10, Fixnum recompile + v72:Fixnum = FixnumMult v71, v43 + Jump bb6(v25, v71, v72, v64) + bb6(v49:ObjectSubclass[class_exact*:Object@VALUE(0x1008)], v50:BasicObject, v51:NilClass|Fixnum, v52:Fixnum[1]): PatchPoint MethodRedefined(Integer@0x1048, +@0x1080, cme:0x1088) - v74:Fixnum = GuardType v49, Fixnum recompile - v75:Fixnum = GuardType v50, Fixnum - v76:Fixnum = FixnumAdd v74, v75 + v75:Fixnum = GuardType v50, Fixnum recompile + v76:Fixnum = GuardType v51, Fixnum + v77:Fixnum = FixnumAdd v75, v76 CheckInterrupts PopInlineFrame - Return v76 + Return v77 "); } @@ -19033,12 +19033,12 @@ mod hir_opt_tests { v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v27 (0x1040), v10, v16, v18 PatchPoint MethodRedefined(Integer@0x1048, +@0x1050, cme:0x1058) - v54:Fixnum = GuardType v10, Fixnum recompile - v55:Fixnum = FixnumAdd v54, v16 - v59:Fixnum = FixnumAdd v55, v18 + v55:Fixnum = GuardType v10, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v16 + v60:Fixnum = FixnumAdd v56, v18 CheckInterrupts PopInlineFrame - Return v59 + Return v60 "); } @@ -19087,11 +19087,11 @@ mod hir_opt_tests { PatchPoint MethodRedefined(Object@0x1008, with_yield@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile PushInlineFrame v25 (0x1040), v10 - v33:BasicObject = InvokeBlock v10 # SendFallbackReason: InvokeBlock: not yet specialized + v34:BasicObject = InvokeBlock v10 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v33 + Return v34 "); } @@ -19138,26 +19138,26 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, with_block_param@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v52:NilClass = Const Value(nil) + v53:NilClass = Const Value(nil) PushInlineFrame v25 (0x1040), v10 - v35:CPtr = GetEP 0 - v36:CUInt64 = LoadField v35, :VM_ENV_DATA_INDEX_FLAGS@0x1048 - v37:CBool = IsBlockParamModified v36 - CondBranch v37, bb6(), bb7() + v36:CPtr = GetEP 0 + v37:CUInt64 = LoadField v36, :VM_ENV_DATA_INDEX_FLAGS@0x1048 + v38:CBool = IsBlockParamModified v37 + CondBranch v38, bb6(), bb7() bb6(): - v39:BasicObject = LoadField v35, :block@0x1049 - Jump bb8(v39, v39) + v40:BasicObject = LoadField v36, :block@0x1049 + Jump bb8(v40, v40) bb7(): - v41:CInt64 = LoadField v35, :VM_ENV_DATA_INDEX_SPECVAL@0x104a - v42:CInt64 = GuardAnyBitSet v41, CUInt64(1) recompile - v43:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) - Jump bb8(v43, v52) - bb8(v33:BasicObject, v34:BasicObject): - v47:BasicObject = Send v33, :call, v10 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall + v42:CInt64 = LoadField v36, :VM_ENV_DATA_INDEX_SPECVAL@0x104a + v43:CInt64 = GuardAnyBitSet v42, CUInt64(1) recompile + v44:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) + Jump bb8(v44, v53) + bb8(v34:BasicObject, v35:BasicObject): + v48:BasicObject = Send v34, :call, v10 # SendFallbackReason: SendWithoutBlock: unsupported optimized method type BlockCall CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v47 + Return v48 "); } @@ -19202,26 +19202,26 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): PatchPoint MethodRedefined(Object@0x1008, callee@0x1010, cme:0x1018) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v53:NilClass = Const Value(nil) + v54:NilClass = Const Value(nil) PushInlineFrame v25 (0x1040), v10 - v37:CPtr = GetEP 0 - v38:CUInt64 = LoadField v37, :VM_ENV_DATA_INDEX_FLAGS@0x1048 - v39:CBool = IsBlockParamModified v38 - CondBranch v39, bb6(), bb7() + v38:CPtr = GetEP 0 + v39:CUInt64 = LoadField v38, :VM_ENV_DATA_INDEX_FLAGS@0x1048 + v40:CBool = IsBlockParamModified v39 + CondBranch v40, bb6(), bb7() bb6(): - v41:BasicObject = LoadField v37, :block@0x1049 - Jump bb8(v41, v41) + v42:BasicObject = LoadField v38, :block@0x1049 + Jump bb8(v42, v42) bb7(): - v43:CInt64 = LoadField v37, :VM_ENV_DATA_INDEX_SPECVAL@0x104a - v44:CInt64 = GuardAnyBitSet v43, CUInt64(1) recompile - v45:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) - Jump bb8(v45, v53) - bb8(v35:BasicObject, v36:BasicObject): - v48:BasicObject = Send v25, &block, :inner, v10, v35 # SendFallbackReason: Complex argument passing + v44:CInt64 = LoadField v38, :VM_ENV_DATA_INDEX_SPECVAL@0x104a + v45:CInt64 = GuardAnyBitSet v44, CUInt64(1) recompile + v46:ObjectSubclass[BlockParamProxy] = Const Value(VALUE(0x1050)) + Jump bb8(v46, v54) + bb8(v36:BasicObject, v37:BasicObject): + v49:BasicObject = Send v25, &block, :inner, v10, v36 # SendFallbackReason: Complex argument passing CheckInterrupts PopInlineFrame PatchPoint NoEPEscape(test) - Return v48 + Return v49 "); } @@ -19281,18 +19281,18 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v91 (0x1068), v17, v19 - v122:CShape = LoadField v91, :shape_id@0x1070 - v123:CShape[0x1071] = GuardBitEquals v122, CShape(0x1071) recompile + v123:CShape = LoadField v91, :shape_id@0x1070 + v124:CShape[0x1071] = GuardBitEquals v123, CShape(0x1071) recompile StoreField v91, :@x@0x1072, v17 WriteBarrier v91, v17 - v126:CShape[0x1073] = Const CShape(0x1073) - StoreField v91, :shape_id@0x1070, v126 + v127:CShape[0x1073] = Const CShape(0x1073) + StoreField v91, :shape_id@0x1070, v127 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode StoreField v91, :@y@0x1074, v19 WriteBarrier v91, v19 - v141:CShape[0x1075] = Const CShape(0x1075) - StoreField v91, :shape_id@0x1070, v141 + v142:CShape[0x1075] = Const CShape(0x1075) + StoreField v91, :shape_id@0x1070, v142 CheckInterrupts PopInlineFrame PatchPoint SingleRactorMode @@ -19306,55 +19306,55 @@ mod hir_opt_tests { PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, initialize@0x1038, cme:0x1040) PushInlineFrame v101 (0x1068), v52, v54 - v161:CShape = LoadField v101, :shape_id@0x1070 - v162:CShape[0x1071] = GuardBitEquals v161, CShape(0x1071) recompile + v163:CShape = LoadField v101, :shape_id@0x1070 + v164:CShape[0x1071] = GuardBitEquals v163, CShape(0x1071) recompile StoreField v101, :@x@0x1072, v52 WriteBarrier v101, v52 - v165:CShape[0x1073] = Const CShape(0x1073) - StoreField v101, :shape_id@0x1070, v165 + v167:CShape[0x1073] = Const CShape(0x1073) + StoreField v101, :shape_id@0x1070, v167 PatchPoint NoEPEscape(initialize) PatchPoint SingleRactorMode StoreField v101, :@y@0x1074, v54 WriteBarrier v101, v54 - v180:CShape[0x1075] = Const CShape(0x1075) - StoreField v101, :shape_id@0x1070, v180 + v182:CShape[0x1075] = Const CShape(0x1075) + StoreField v101, :shape_id@0x1070, v182 CheckInterrupts PopInlineFrame PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, ==@0x1080, cme:0x1088) PushInlineFrame v91 (0x1068), v101 PatchPoint SingleRactorMode - v198:CShape = LoadField v91, :shape_id@0x1070 - v199:CShape[0x1075] = GuardBitEquals v198, CShape(0x1075) recompile - v200:BasicObject = LoadField v91, :@x@0x1072 + v201:CShape = LoadField v91, :shape_id@0x1070 + v202:CShape[0x1075] = GuardBitEquals v201, CShape(0x1075) recompile + v203:BasicObject = LoadField v91, :@x@0x1072 PatchPoint NoEPEscape(==) PatchPoint MethodRedefined(Point@0x1008, x@0x10b0, cme:0x10b8) PatchPoint MethodRedefined(Integer@0x10e0, ==@0x1080, cme:0x10e8) - v255:Fixnum = GuardType v200, Fixnum recompile - v257:BoolExact = FixnumEq v255, v52 - v212:CBool = Test v257 - v213:FalseClass = RefineType v257, Falsy - CondBranch v212, bb19(), bb18(v91, v101, v213) + v258:Fixnum = GuardType v203, Fixnum recompile + v260:BoolExact = FixnumEq v258, v52 + v215:CBool = Test v260 + v216:FalseClass = RefineType v260, Falsy + CondBranch v215, bb19(), bb18(v91, v101, v216) bb19(): PatchPoint SingleRactorMode - v220:CShape = LoadField v91, :shape_id@0x1070 - v221:CShape[0x1075] = GuardBitEquals v220, CShape(0x1075) recompile - v222:BasicObject = LoadField v91, :@y@0x1074 + v223:CShape = LoadField v91, :shape_id@0x1070 + v224:CShape[0x1075] = GuardBitEquals v223, CShape(0x1075) recompile + v225:BasicObject = LoadField v91, :@y@0x1074 PatchPoint NoEPEscape(==) PatchPoint NoSingletonClass(Point@0x1008) PatchPoint MethodRedefined(Point@0x1008, y@0x1110, cme:0x1118) - v262:CShape = LoadField v101, :shape_id@0x1070 - v263:CShape[0x1075] = GuardBitEquals v262, CShape(0x1075) recompile - v264:BasicObject = LoadField v101, :@y@0x1074 + v265:CShape = LoadField v101, :shape_id@0x1070 + v266:CShape[0x1075] = GuardBitEquals v265, CShape(0x1075) recompile + v267:BasicObject = LoadField v101, :@y@0x1074 PatchPoint MethodRedefined(Integer@0x10e0, ==@0x1080, cme:0x10e8) - v267:Fixnum = GuardType v222, Fixnum recompile - v268:Fixnum = GuardType v264, Fixnum - v269:BoolExact = FixnumEq v267, v268 - Jump bb18(v91, v101, v269) - bb18(v232:ObjectSubclass[class_exact:Point], v233:ObjectSubclass[class_exact:Point], v234:BoolExact): + v270:Fixnum = GuardType v225, Fixnum recompile + v271:Fixnum = GuardType v267, Fixnum + v272:BoolExact = FixnumEq v270, v271 + Jump bb18(v91, v101, v272) + bb18(v235:ObjectSubclass[class_exact:Point], v236:ObjectSubclass[class_exact:Point], v237:BoolExact): CheckInterrupts PopInlineFrame - Return v234 + Return v237 "); } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index a61aa4c11bb7a3..02de128ef4ee32 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -136,14 +136,14 @@ mod snapshot_tests { v23:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v13, v15, v11], locals: [] } PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) v25:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile - v43:Fixnum[0] = Const Value(0) + v44:Fixnum[0] = Const Value(0) PushInlineFrame v25 (0x1048), v13, v15, v11 - v37:Any = Snapshot FrameState { pc: 0x1050, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v43], caller: v23 } - v38:ArrayExact = NewArray v13, v15, v11 - v39:Any = Snapshot FrameState { pc: 0x1058, stack: [v38], locals: [a=v13, b=v15, c=v11, ID(0)=v43], caller: v23 } + v38:Any = Snapshot FrameState { pc: 0x1050, stack: [v13, v15, v11], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } + v39:ArrayExact = NewArray v13, v15, v11 + v40:Any = Snapshot FrameState { pc: 0x1058, stack: [v39], locals: [a=v13, b=v15, c=v11, ID(0)=v44], caller: v27 } CheckInterrupts PopInlineFrame - Return v38 + Return v39 "); } @@ -175,14 +175,14 @@ mod snapshot_tests { v14:Any = Snapshot FrameState { pc: 0x1008, stack: [v6, v11, v13], locals: [] } PatchPoint MethodRedefined(Object@0x1010, foo@0x1018, cme:0x1020) v22:ObjectSubclass[class_exact*:Object@VALUE(0x1010)] = GuardType v6, ObjectSubclass[class_exact*:Object@VALUE(0x1010)] recompile - v38:Fixnum[0] = Const Value(0) + v39:Fixnum[0] = Const Value(0) PushInlineFrame v22 (0x1048), v11, v13 - v32:Any = Snapshot FrameState { pc: 0x1050, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v38], caller: v14 } - v33:ArrayExact = NewArray v11, v13 - v34:Any = Snapshot FrameState { pc: 0x1058, stack: [v33], locals: [a=v11, b=v13, ID(0)=v38], caller: v14 } + v33:Any = Snapshot FrameState { pc: 0x1050, stack: [v11, v13], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } + v34:ArrayExact = NewArray v11, v13 + v35:Any = Snapshot FrameState { pc: 0x1058, stack: [v34], locals: [a=v11, b=v13, ID(0)=v39], caller: v24 } CheckInterrupts PopInlineFrame - Return v33 + Return v34 "); }