diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index f678eabd1c2ba4..0c3fce5bccb173 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -221,7 +221,6 @@ jobs: - { uses: './.github/actions/compilers', name: 'NDEBUG', with: { cppflags: '-DNDEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'RUBY_DEBUG', with: { cppflags: '-DRUBY_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'ARRAY_DEBUG', with: { cppflags: '-DARRAY_DEBUG' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'BIGNUM_DEBUG', with: { cppflags: '-DBIGNUM_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'CCAN_LIST_DEBUG', with: { cppflags: '-DCCAN_LIST_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'CPDEBUG=-1', with: { cppflags: '-DCPDEBUG=-1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'ENC_DEBUG', with: { cppflags: '-DENC_DEBUG' }, timeout-minutes: 5 } @@ -259,8 +258,6 @@ jobs: with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github, persist-credentials: false } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'USE_LAZY_LOAD', with: { cppflags: '-DUSE_LAZY_LOAD' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'USE_SYMBOL_GC=0', with: { cppflags: '-DUSE_SYMBOL_GC=0' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'USE_THREAD_CACHE=0', with: { cppflags: '-DUSE_THREAD_CACHE=0' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'USE_RUBY_DEBUG_LOG=1', with: { cppflags: '-DUSE_RUBY_DEBUG_LOG=1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'USE_DEBUG_COUNTER', with: { cppflags: '-DUSE_DEBUG_COUNTER=1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'SHARABLE_MIDDLE_SUBSTRING', with: { cppflags: '-DSHARABLE_MIDDLE_SUBSTRING=1' }, timeout-minutes: 5 } diff --git a/bignum.c b/bignum.c index f5c58ab74312a4..11dff4d13927cf 100644 --- a/bignum.c +++ b/bignum.c @@ -2943,11 +2943,6 @@ bary_divmod(BDIGIT *qds, size_t qn, BDIGIT *rds, size_t rn, const BDIGIT *xds, s } } - -#ifndef BIGNUM_DEBUG -# define BIGNUM_DEBUG (0+RUBY_DEBUG) -#endif - static int bigzero_p(VALUE x) { diff --git a/cont.c b/cont.c index 36ee34fe670f8b..e5239635081629 100644 --- a/cont.c +++ b/cont.c @@ -41,6 +41,7 @@ extern int madvise(caddr_t, size_t, int); #include "vm_sync.h" #include "id_table.h" #include "ractor_core.h" +#include "zjit.h" enum { DEBUG = 0, @@ -1455,8 +1456,11 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) { - callback(cfp->iseq, data); + if (CFP_PC(cfp) && CFP_ISEQ(cfp)) { + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { + callback(iseq, data); + } } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } @@ -1476,7 +1480,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->jit_return && cfp->jit_return != leave_exception) { + if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { ((rb_control_frame_t *)cfp)->jit_return = leave_exit; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1576,8 +1580,8 @@ show_vm_pcs(const rb_control_frame_t *cfp, int i=0; while (cfp != end_of_cfp) { int pc = 0; - if (cfp->iseq) { - pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + if (CFP_ISEQ(cfp)) { + pc = cfp->pc - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; } fprintf(stderr, "%2d pc: %d\n", i++, pc); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/depend b/depend index f22dbbd8189aa0..eef68cd3fcb09d 100644 --- a/depend +++ b/depend @@ -2620,6 +2620,7 @@ cont.$(OBJEXT): {$(VPATH)}vm_debug.h cont.$(OBJEXT): {$(VPATH)}vm_opts.h cont.$(OBJEXT): {$(VPATH)}vm_sync.h cont.$(OBJEXT): {$(VPATH)}yjit.h +cont.$(OBJEXT): {$(VPATH)}zjit.h debug.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h debug.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h debug.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5420,6 +5421,7 @@ error.$(OBJEXT): {$(VPATH)}vm_opts.h error.$(OBJEXT): {$(VPATH)}vm_sync.h error.$(OBJEXT): {$(VPATH)}warning.rbinc error.$(OBJEXT): {$(VPATH)}yjit.h +error.$(OBJEXT): {$(VPATH)}zjit.h eval.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h eval.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h eval.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5689,6 +5691,7 @@ eval.$(OBJEXT): {$(VPATH)}vm_core.h eval.$(OBJEXT): {$(VPATH)}vm_debug.h eval.$(OBJEXT): {$(VPATH)}vm_opts.h eval.$(OBJEXT): {$(VPATH)}vm_sync.h +eval.$(OBJEXT): {$(VPATH)}zjit.h explicit_bzero.$(OBJEXT): {$(VPATH)}config.h explicit_bzero.$(OBJEXT): {$(VPATH)}explicit_bzero.c explicit_bzero.$(OBJEXT): {$(VPATH)}internal/attr/format.h @@ -7827,6 +7830,7 @@ jit.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h jit.$(OBJEXT): $(top_srcdir)/internal/bits.h jit.$(OBJEXT): $(top_srcdir)/internal/box.h jit.$(OBJEXT): $(top_srcdir)/internal/class.h +jit.$(OBJEXT): $(top_srcdir)/internal/compile.h jit.$(OBJEXT): $(top_srcdir)/internal/compilers.h jit.$(OBJEXT): $(top_srcdir)/internal/fixnum.h jit.$(OBJEXT): $(top_srcdir)/internal/gc.h @@ -8072,6 +8076,7 @@ jit.$(OBJEXT): {$(VPATH)}vm_core.h jit.$(OBJEXT): {$(VPATH)}vm_debug.h jit.$(OBJEXT): {$(VPATH)}vm_opts.h jit.$(OBJEXT): {$(VPATH)}vm_sync.h +jit.$(OBJEXT): {$(VPATH)}zjit.h load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h load.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -18617,6 +18622,7 @@ thread.$(OBJEXT): {$(VPATH)}vm_core.h thread.$(OBJEXT): {$(VPATH)}vm_debug.h thread.$(OBJEXT): {$(VPATH)}vm_opts.h thread.$(OBJEXT): {$(VPATH)}vm_sync.h +thread.$(OBJEXT): {$(VPATH)}zjit.h time.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h time.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h time.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20212,6 +20218,7 @@ vm_backtrace.$(OBJEXT): {$(VPATH)}vm_core.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_debug.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_opts.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_sync.h +vm_backtrace.$(OBJEXT): {$(VPATH)}zjit.h vm_dump.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_dump.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_dump.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20460,6 +20467,7 @@ vm_dump.$(OBJEXT): {$(VPATH)}vm_core.h vm_dump.$(OBJEXT): {$(VPATH)}vm_debug.h vm_dump.$(OBJEXT): {$(VPATH)}vm_dump.c vm_dump.$(OBJEXT): {$(VPATH)}vm_opts.h +vm_dump.$(OBJEXT): {$(VPATH)}zjit.h vm_sync.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_sync.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_sync.$(OBJEXT): $(CCAN_DIR)/list/list.h diff --git a/error.c b/error.c index 140049fdd2cc2b..52bd3629bf2d13 100644 --- a/error.c +++ b/error.c @@ -50,6 +50,7 @@ #include "ruby_assert.h" #include "vm_core.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" @@ -908,6 +909,10 @@ bug_report_file(const char *file, int line, rb_pid_t *pid) int len = err_position_0(buf, sizeof(buf), file, line); if (out) { + /* Disable buffering so crash report output is not lost if + * rb_vm_bugreport() triggers a secondary crash (e.g. SIGSEGV + * while walking JIT frames). */ + setvbuf(out, NULL, _IONBF, 0); if ((ssize_t)fwrite(buf, 1, len, out) == (ssize_t)len) return out; fclose(out); } @@ -1081,6 +1086,10 @@ die(void) _set_abort_behavior( 0, _CALL_REPORTFAULT); #endif + /* Reset SIGABRT to default so that abort() does not trigger our custom + * handler (sigabrt), which would re-open the crash report file with "w" + * and truncate the report already written by rb_bug(). */ + signal(SIGABRT, SIG_DFL); abort(); } @@ -2367,7 +2376,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method) rb_ivar_set(exc, id_name, method); err_init_recv(exc, recv); if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_ivar_set(exc, id_iseq, rb_iseqw_new(cfp->iseq)); + rb_ivar_set(exc, id_iseq, rb_iseqw_new(CFP_ISEQ(cfp))); } return exc; } diff --git a/eval.c b/eval.c index 7d5ae75e3e6ee9..e3630c246ae1f0 100644 --- a/eval.c +++ b/eval.c @@ -37,6 +37,7 @@ #include "ruby/vm.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" NORETURN(static void rb_raise_jump(VALUE, VALUE)); void rb_ec_clear_current_thread_trace_func(const rb_execution_context_t *ec); @@ -2010,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec) while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { if (VM_FRAME_RUBYFRAME_P(cfp)) { - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } - else if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_ENSURE && + else if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_ENSURE && !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) && !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; diff --git a/gc.c b/gc.c index eb949bfae7c492..d6d517d6a44c9e 100644 --- a/gc.c +++ b/gc.c @@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && cfp->pc) { - const VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; - RUBY_ASSERT(cfp->pc >= iseq_encoded, "PC not set when allocating, breaking tracing"); - RUBY_ASSERT(cfp->pc <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size; + RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } diff --git a/jit.c b/jit.c index 11ff015e362c3d..10c845491b2271 100644 --- a/jit.c +++ b/jit.c @@ -12,6 +12,7 @@ #include "insns.inc" #include "insns_info.inc" #include "iseq.h" +#include "internal/compile.h" #include "internal/gc.h" #include "vm_sync.h" #include "internal/fixnum.h" @@ -19,6 +20,7 @@ #include "internal/class.h" #include "internal/imemo.h" #include "ruby/internal/core/rtypeddata.h" +#include "zjit.h" #ifndef _WIN32 #include @@ -82,6 +84,19 @@ rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) return rb_vm_insn_addr2opcode((const void *)at_pc); } +// Get the bare opcode given a program counter. Always returns the base +// instruction, stripping trace/zjit variants. +int +rb_iseq_bare_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) +{ + if (OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE) { + RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED)); + } + + const VALUE at_pc = *pc; + return rb_vm_insn_addr2insn((const void *)at_pc); +} + unsigned long rb_RSTRING_LEN(VALUE str) { @@ -388,7 +403,7 @@ rb_get_ec_cfp(const rb_execution_context_t *ec) const rb_iseq_t * rb_get_cfp_iseq(struct rb_control_frame_struct *cfp) { - return cfp->iseq; + return CFP_ISEQ(cfp); } VALUE * diff --git a/lib/prism/translation/ripper/shim.rb b/lib/prism/translation/ripper/shim.rb index 10e21cd16a213f..038235e078f2d8 100644 --- a/lib/prism/translation/ripper/shim.rb +++ b/lib/prism/translation/ripper/shim.rb @@ -2,4 +2,4 @@ # This writes the prism ripper translation into the Ripper constant so that # users can transparently use Ripper without any changes. -Ripper = Prism::Translation::Ripper +Ripper = Prism::Translation::Ripper # :nodoc: diff --git a/symbol.c b/symbol.c index 85340be3a74c27..d3d7e13ea43626 100644 --- a/symbol.c +++ b/symbol.c @@ -27,13 +27,6 @@ #include "builtin.h" #include "ruby/internal/attr/nonstring.h" -#if defined(USE_SYMBOL_GC) && !(USE_SYMBOL_GC+0) -# undef USE_SYMBOL_GC -# define USE_SYMBOL_GC 0 -#else -# undef USE_SYMBOL_GC -# define USE_SYMBOL_GC 1 -#endif #if defined(SYMBOL_DEBUG) && (SYMBOL_DEBUG+0) # undef SYMBOL_DEBUG # define SYMBOL_DEBUG 1 diff --git a/thread.c b/thread.c index b13b06d7e5d94e..f876b4bd05c80e 100644 --- a/thread.c +++ b/thread.c @@ -99,6 +99,7 @@ #include "ractor_core.h" #include "vm_debug.h" #include "vm_sync.h" +#include "zjit.h" #include "ccan/list/list.h" @@ -5905,7 +5906,7 @@ static void update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(cfp->iseq); + VALUE coverage = rb_iseq_coverage(CFP_ISEQ(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE lines = RARRAY_AREF(coverage, COVERAGE_INDEX_LINES); if (lines) { @@ -5915,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) VALUE num; void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset); if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) { - rb_iseq_clear_event_flags(cfp->iseq, cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); + rb_iseq_clear_event_flags(CFP_ISEQ(cfp), CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); rb_ary_push(lines, LONG2FIX(line + 1)); return; } @@ -5936,12 +5937,12 @@ static void update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(cfp->iseq); + VALUE coverage = rb_iseq_coverage(CFP_ISEQ(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES); if (branches) { - long pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; - long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(cfp->iseq), pc)), count; + long pc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded - 1; + long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(CFP_ISEQ(cfp)), pc)), count; VALUE counters = RARRAY_AREF(branches, 1); VALUE num = RARRAY_AREF(counters, idx); count = FIX2LONG(num) + 1; diff --git a/vm.c b/vm.c index 3825189f38ebac..0398b9f74c9683 100644 --- a/vm.c +++ b/vm.c @@ -516,7 +516,7 @@ rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) static inline rb_jit_func_t yjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); // Increment the ISEQ's call counter and trigger JIT compilation if not compiled @@ -536,7 +536,7 @@ yjit_compile(rb_execution_context_t *ec) static inline rb_jit_func_t zjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); if (body->jit_entry == NULL) { @@ -559,6 +559,8 @@ zjit_compile(rb_execution_context_t *ec) # define zjit_compile(ec) ((rb_jit_func_t)0) #endif +static inline void zjit_materialize_frames(rb_control_frame_t *cfp); + // Execute JIT code compiled by yjit_compile() or zjit_compile() static inline VALUE jit_exec(rb_execution_context_t *ec) @@ -578,7 +580,15 @@ jit_exec(rb_execution_context_t *ec) if (zjit_entry) { rb_jit_func_t func = zjit_compile(ec); if (func) { - return ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + // Materialize any remaining lightweight ZJIT frames on side exit. + // This is done here (once per JIT entry) instead of in each side exit + // to reduce generated code size. + if (UNDEF_P(result)) { + ec->cfp->jit_return = 0; + zjit_materialize_frames(ec->cfp); + } + return result; } } #endif @@ -595,7 +605,7 @@ jit_exec(rb_execution_context_t *ec) static inline rb_jit_func_t jit_compile_exception(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); #if USE_ZJIT @@ -931,7 +941,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (cfp->iseq) { + if (CFP_ISEQ(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1109,13 +1119,14 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } else { - local_size = ISEQ_BODY(cfp->iseq)->local_table_size; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int ci_offset = local_size - ISEQ_BODY(cfp->iseq)->param.size + VM_ENV_DATA_SIZE; + local_size = ISEQ_BODY(iseq)->local_table_size; + if (ISEQ_BODY(iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int ci_offset = local_size - ISEQ_BODY(iseq)->param.size + VM_ENV_DATA_SIZE; CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset]; local_size += vm_ci_argc(ci); @@ -1127,8 +1138,8 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co // This is done before creating the imemo_env because VM_STACK_ENV_WRITE // below leaves the on-stack ep in a state that is unsafe to GC. if (VM_FRAME_RUBYFRAME_P(cfp)) { - rb_yjit_invalidate_ep_is_bp(cfp->iseq); - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_yjit_invalidate_ep_is_bp(iseq); + rb_zjit_invalidate_no_ep_escape(iseq); } /* @@ -1156,7 +1167,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co env_ep = &env_body[local_size - 1 /* specval */]; env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; - env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? iseq : NULL); env->ep = env_ep; env->env = env_body; env->env_size = env_size; @@ -1678,8 +1689,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, CFP_ISEQ(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(CFP_ISEQ(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1986,9 +1997,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (cfp->pc == 0 || cfp->iseq == 0) { + while (!CFP_PC(cfp) || !CFP_ISEQ(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + struct vm_ifunc *ifunc = (struct vm_ifunc *)CFP_ISEQ(cfp); return ifunc->svar_lep; } else { @@ -2071,7 +2082,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(cfp->iseq)); + return RSTRING_PTR(rb_iseq_path(CFP_ISEQ(cfp))); } else { return 0; @@ -2100,7 +2111,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(cfp->iseq); + return rb_iseq_path(CFP_ISEQ(cfp)); } else { if (pline) *pline = 0; @@ -2582,7 +2593,7 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); rb_hook_list_t *local_hooks = NULL; unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; if (RB_UNLIKELY(local_hooks_cnt > 0)) { @@ -2835,6 +2846,32 @@ vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, return result; } +static inline void +zjit_materialize_frames(rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return; + + while (true) { + if (CFP_JIT_RETURN(cfp)) { + const zjit_jit_frame_t *jit_frame = (const zjit_jit_frame_t *)cfp->jit_return; + cfp->pc = jit_frame->pc; + cfp->_iseq = (rb_iseq_t *)jit_frame->iseq; + if (jit_frame->materialize_block_code) { + cfp->block_code = NULL; + } + cfp->jit_return = 0; + } + if (VM_FRAME_FINISHED_P(cfp)) break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } +} + +void +rb_zjit_materialize_frames(rb_control_frame_t *cfp) +{ + zjit_materialize_frames(cfp); +} + static inline VALUE vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) { @@ -2852,7 +2889,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) { + while (CFP_PC(ec->cfp) == 0 || CFP_ISEQ(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2866,7 +2903,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + epc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2879,7 +2916,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2906,13 +2943,14 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* TAG_BREAK */ *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; + zjit_materialize_frames(cfp); return Qundef; } } } if (state == TAG_RAISE) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2928,7 +2966,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2943,7 +2981,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const rb_control_frame_t *escape_cfp; escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -2961,7 +3000,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -2973,7 +3012,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V break; } else if (entry->type == type) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -2987,7 +3027,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -3007,8 +3047,9 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); + zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->_iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -3630,8 +3671,18 @@ rb_execution_context_update(rb_execution_context_t *ec) while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); - cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); - cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + if (CFP_JIT_RETURN(cfp)) { + rb_zjit_jit_frame_update_references((zjit_jit_frame_t *)cfp->jit_return); + // block_code must always be relocated. For ISEQ frames, the JIT caller + // may have written it (gen_block_handler_specval) for passing blocks. + // For C frames, rb_iterate0 may have written an ifunc to block_code + // after the JIT pushed the frame. NULL is safe to pass to rb_gc_location. + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } + else { + cfp->_iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->_iseq); + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } if (!VM_ENV_LOCAL_P(ep)) { const VALUE *prev_ep = VM_ENV_PREV_EP(ep); @@ -3682,7 +3733,9 @@ rb_execution_context_mark(const rb_execution_context_t *ec) VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)cfp->iseq); + rb_gc_mark_movable((VALUE)CFP_ISEQ(cfp)); + // Mark block_code directly (not through rb_zjit_cfp_block_code) + // because rb_iterate0 may write a valid ifunc after JIT frame push. rb_gc_mark_movable((VALUE)cfp->block_code); if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) { @@ -4522,7 +4575,7 @@ Init_VM(void) rb_root_fiber_obj_setup(th); rb_vm_register_global_object((VALUE)iseq); - th->ec->cfp->iseq = iseq; + th->ec->cfp->_iseq = iseq; th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded; th->ec->cfp->self = th->top_self; @@ -4558,7 +4611,7 @@ rb_vm_set_progname(VALUE filename) --cfp; filename = rb_str_new_frozen(filename); - rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq)); + rb_iseq_pathobj_set(CFP_ISEQ(cfp), filename, rb_iseq_realpath(CFP_ISEQ(cfp))); } extern const struct st_hash_type rb_fstring_hash_type; @@ -4890,7 +4943,7 @@ vm_analysis_operand(int insn, int n, VALUE op) HASH_ASET(ihash, INT2FIX(n), ophash); } /* intern */ - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); /* set count */ if (NIL_P(cv = rb_hash_aref(ophash, valstr))) { @@ -5092,7 +5145,7 @@ vm_collect_usage_operand(int insn, int n, VALUE op) if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) { VALUE valstr; - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); RB_GC_GUARD(valstr); diff --git a/vm_backtrace.c b/vm_backtrace.c index 07d2e33e321787..c0bc46b8caf5c7 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -18,6 +18,7 @@ #include "ruby/debug.h" #include "ruby/encoding.h" #include "vm_core.h" +#include "zjit.h" static VALUE rb_cBacktrace; static VALUE rb_cBacktraceLocation; @@ -101,9 +102,9 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && cfp->iseq) { - const rb_iseq_t *iseq = cfp->iseq; - int line = calc_lineno(iseq, cfp->pc); + if (VM_FRAME_RUBYFRAME_P(cfp) && CFP_ISEQ(cfp)) { + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + int line = calc_lineno(iseq, CFP_PC(cfp)); if (line != 0) { return line; } @@ -617,7 +618,7 @@ backtrace_size(const rb_execution_context_t *ec) static bool is_rescue_or_ensure_frame(const rb_control_frame_t *cfp) { - enum rb_iseq_type type = ISEQ_BODY(cfp->iseq)->type; + enum rb_iseq_type type = ISEQ_BODY(CFP_ISEQ(cfp))->type; return type == ISEQ_TYPE_RESCUE || type == ISEQ_TYPE_ENSURE; } @@ -687,17 +688,17 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq) { - if (cfp->pc) { + if (CFP_ISEQ(cfp)) { + if (CFP_PC(cfp)) { if (start_frame > 0) { start_frame--; } else { - bool internal = is_internal_location(cfp->iseq); + bool internal = is_internal_location(CFP_ISEQ(cfp)); if (skip_internal && internal) continue; if (!skip_next_frame) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + const VALUE *pc = CFP_PC(cfp); if (internal && backpatch_counter > 0) { // To keep only one internal frame, discard the previous backpatch frames bt->backtrace_size -= backpatch_counter; @@ -752,10 +753,10 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq && cfp->pc && !(skip_internal && is_internal_location(cfp->iseq))) { + if (CFP_ISEQ(cfp) && CFP_PC(cfp) && !(skip_internal && is_internal_location(CFP_ISEQ(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc - bt_backpatch_loc(backpatch_counter, loc, cfp->iseq, cfp->pc); - RB_OBJ_WRITTEN(btobj, Qundef, cfp->iseq); + bt_backpatch_loc(backpatch_counter, loc, CFP_ISEQ(cfp), CFP_PC(cfp)); + RB_OBJ_WRITTEN(btobj, Qundef, CFP_ISEQ(cfp)); if (do_yield) { bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); } @@ -1019,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (cfp->iseq) { - if (cfp->pc) { + if (CFP_ISEQ(cfp)) { + if (CFP_PC(cfp)) { iter_iseq(arg, cfp); } } @@ -1052,8 +1053,8 @@ oldbt_init(void *ptr, size_t dmy) static void oldbt_iter_iseq(void *ptr, const rb_control_frame_t *cfp) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + const VALUE *pc = CFP_PC(cfp); struct oldbt_arg *arg = (struct oldbt_arg *)ptr; VALUE file = arg->filename = rb_iseq_path(iseq); VALUE name = ISEQ_BODY(iseq)->location.label; @@ -1550,17 +1551,18 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) { struct collect_caller_bindings_data *data = (struct collect_caller_bindings_data *)arg; VALUE frame = rb_ary_new2(6); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); rb_ary_store(frame, CALLER_BINDING_SELF, cfp->self); rb_ary_store(frame, CALLER_BINDING_CLASS, get_klass(cfp)); rb_ary_store(frame, CALLER_BINDING_BINDING, GC_GUARDED_PTR(cfp)); /* create later */ - rb_ary_store(frame, CALLER_BINDING_ISEQ, cfp->iseq ? (VALUE)cfp->iseq : Qnil); + rb_ary_store(frame, CALLER_BINDING_ISEQ, iseq ? (VALUE)iseq : Qnil); rb_ary_store(frame, CALLER_BINDING_CFP, GC_GUARDED_PTR(cfp)); rb_backtrace_location_t *loc = &data->bt->backtrace[data->bt->backtrace_size++]; RB_OBJ_WRITE(data->btobj, &loc->cme, rb_vm_frame_method_entry(cfp)); - RB_OBJ_WRITE(data->btobj, &loc->iseq, cfp->iseq); - loc->pc = cfp->pc; + RB_OBJ_WRITE(data->btobj, &loc->iseq, iseq); + loc->pc = CFP_PC(cfp); VALUE vloc = location_create(loc, (void *)data->btobj); rb_ary_store(frame, CALLER_BINDING_LOC, vloc); @@ -1745,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; ipc != 0) { + if (VM_FRAME_RUBYFRAME_P_UNCHECKED(cfp) && CFP_PC(cfp)) { if (start > 0) { start--; continue; @@ -1753,17 +1755,18 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b /* record frame info */ cme = rb_vm_frame_method_entry_unchecked(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (cme && cme->def->type == VM_METHOD_TYPE_ISEQ) { buff[i] = (VALUE)cme; } else { - buff[i] = (VALUE)cfp->iseq; + buff[i] = (VALUE)iseq; } if (lines) { - const VALUE *pc = cfp->pc; - VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - VALUE *pc_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; + const VALUE *pc = CFP_PC(cfp); + VALUE *iseq_encoded = ISEQ_BODY(iseq)->iseq_encoded; + VALUE *pc_end = iseq_encoded + ISEQ_BODY(iseq)->iseq_size; // The topmost frame may have an invalid PC because the JIT // may leave it uninitialized for speed. JIT code must update the PC @@ -1778,7 +1781,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b lines[i] = 0; } else { - lines[i] = calc_lineno(cfp->iseq, pc); + lines[i] = calc_lineno(iseq, pc); } } diff --git a/vm_core.h b/vm_core.h index 85711ffc7e8113..85664e18b8396b 100644 --- a/vm_core.h +++ b/vm_core.h @@ -920,7 +920,7 @@ struct rb_block { typedef struct rb_control_frame_struct { const VALUE *pc; // cfp[0] VALUE *sp; // cfp[1] - const rb_iseq_t *iseq; // cfp[2] + const rb_iseq_t *_iseq; // cfp[2] -- use rb_cfp_iseq(cfp) to read VALUE self; // cfp[3] / block[0] const VALUE *ep; // cfp[4] / block[1] const void *block_code; // cfp[5] / block[2] -- iseq, ifunc, or forwarded block handler @@ -1532,7 +1532,10 @@ static inline int VM_FRAME_CFRAME_P(const rb_control_frame_t *cfp) { int cframe_p = VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_CFRAME) != 0; - VM_ASSERT(RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || + // With ZJIT lightweight frames, cfp->_iseq may be stale (not yet materialized), + // so skip this assertion when jit_return is set (zjit.h is not available here). + VM_ASSERT(cfp->jit_return || + RUBY_VM_NORMAL_ISEQ_P(cfp->_iseq) != cframe_p || (VM_FRAME_TYPE(cfp) & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY); return cframe_p; } diff --git a/vm_dump.c b/vm_dump.c index 4864ccf58bdb30..4a7b0df5c6c0eb 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -37,6 +37,7 @@ #include "iseq.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" #define MAX_POSBUF 128 @@ -118,21 +119,21 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (cfp->iseq != 0) { + if (CFP_ISEQ(cfp)) { + iseq = CFP_ISEQ(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + if (RUBY_VM_IFUNC_P(iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)iseq)) { + tmp = rb_sym2str((VALUE)iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (CFP_PC(cfp)) { + pc = CFP_PC(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -169,6 +170,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c else { kprintf("b:---- "); } + kprintf("r:%p ", cfp->jit_return); kprintf("%-6s", magic); if (line) { kprintf(" %s", posbuf); @@ -208,7 +210,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c if (ISEQ_BODY(iseq)->local_table_size > 0) { kprintf(" lvars:\n"); for (unsigned int i=0; ilocal_table_size; i++) { - const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1; + const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1; kprintf(" %s: %s\n", rb_id2name(ISEQ_BODY(iseq)->local_table[i]), rb_raw_obj_info(buff, 0x100, argv[i])); @@ -339,21 +341,22 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && cfp->iseq != 0) { + if (cfp && CFP_ISEQ(cfp)) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + const rb_iseq_t *resolved_iseq = CFP_ISEQ(cfp); + if (RUBY_VM_IFUNC_P(resolved_iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)resolved_iseq)) { + tmp = rb_sym2str((VALUE)resolved_iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (CFP_PC(cfp)) { + iseq = resolved_iseq; + pc = CFP_PC(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -564,9 +567,9 @@ static const VALUE * vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - const VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE; + const VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { bp += 1; } return bp; @@ -581,7 +584,7 @@ vm_stack_dump_each(const rb_execution_context_t *ec, const rb_control_frame_t *c const VALUE *ep = cfp->ep; if (VM_FRAME_RUBYFRAME_P(cfp)) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); argc = ISEQ_BODY(iseq)->param.lead_num; local_table_size = ISEQ_BODY(iseq)->local_table_size; } @@ -652,7 +655,7 @@ rb_vmdebug_debug_print_register(const rb_execution_context_t *ec, FILE *errout) ptrdiff_t cfpi; if (VM_FRAME_RUBYFRAME_P(cfp)) { - pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + pc = cfp->pc - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; } if (ep < 0 || (size_t)ep > ec->vm_stack_size) { @@ -677,7 +680,7 @@ rb_vmdebug_thread_dump_regs(VALUE thval, FILE *errout) bool rb_vmdebug_debug_print_pre(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE *_pc, FILE *errout) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (iseq != 0) { ptrdiff_t pc = _pc - ISEQ_BODY(iseq)->iseq_encoded; diff --git a/vm_eval.c b/vm_eval.c index 25d366f5cd19e6..c6d5a1dd44d1ea 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1988,12 +1988,12 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li block.as.captured = *VM_CFP_TO_CAPTURED_BLOCK(cfp); block.as.captured.self = self; - block.as.captured.code.iseq = cfp->iseq; + block.as.captured.code.iseq = CFP_ISEQ(cfp); block.type = block_type_iseq; // EP is not escaped to the heap here, but captured and reused by another frame. // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT. - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_zjit_invalidate_no_ep_escape(CFP_ISEQ(cfp)); iseq = eval_make_iseq(src, file, line, &block); if (!iseq) { @@ -2773,9 +2773,9 @@ rb_f_local_variables(VALUE _) local_var_list_init(&vars); while (cfp) { - if (cfp->iseq) { - for (i = 0; i < ISEQ_BODY(cfp->iseq)->local_table_size; i++) { - local_var_list_add(&vars, ISEQ_BODY(cfp->iseq)->local_table[i]); + if (CFP_ISEQ(cfp)) { + for (i = 0; i < ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size; i++) { + local_var_list_add(&vars, ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i]); } } if (!VM_ENV_LOCAL_P(cfp->ep)) { @@ -2849,10 +2849,11 @@ rb_current_realfilepath(void) rb_control_frame_t *cfp = ec->cfp; cfp = vm_get_ruby_level_caller_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)); if (cfp != NULL) { - VALUE path = rb_iseq_realpath(cfp->iseq); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + VALUE path = rb_iseq_realpath(iseq); if (RTEST(path)) return path; // eval context - path = rb_iseq_path(cfp->iseq); + path = rb_iseq_path(iseq); if (path == eval_default_path) { return Qnil; } @@ -2878,7 +2879,7 @@ struct vm_ifunc * rb_current_ifunc(void) { // Search VM_FRAME_MAGIC_IFUNC to see ifunc imemos put on the iseq field. - VALUE ifunc = (VALUE)GET_EC()->cfp->iseq; + VALUE ifunc = (VALUE)CFP_ISEQ(GET_EC()->cfp); RUBY_ASSERT_ALWAYS(imemo_type_p(ifunc, imemo_ifunc)); return (struct vm_ifunc *)ifunc; } diff --git a/vm_exec.h b/vm_exec.h index 641ace4eaf29b9..f0796e93ab9c18 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -68,10 +68,10 @@ error ! #define INSN_ENTRY_SIG(insn) \ if (0) { \ ruby_debug_printf("exec: %s@(%"PRIdPTRDIFF", %"PRIdPTRDIFF")@%s:%u\n", #insn, \ - (reg_pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded), \ - (reg_cfp->pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded), \ - RSTRING_PTR(rb_iseq_path(reg_cfp->iseq)), \ - rb_iseq_line_no(reg_cfp->iseq, reg_pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded)); \ + (reg_pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded), \ + (reg_cfp->pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded), \ + RSTRING_PTR(rb_iseq_path(CFP_ISEQ(reg_cfp))), \ + rb_iseq_line_no(CFP_ISEQ(reg_cfp), reg_pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded)); \ } #define INSN_DISPATCH_SIG(insn) @@ -189,6 +189,10 @@ default: \ rb_jit_func_t func = zjit_compile(ec); \ if (func) { \ val = zjit_entry(ec, ec->cfp, func); \ + if (UNDEF_P(val)) { \ + ec->cfp->jit_return = 0; \ + zjit_materialize_frames(ec->cfp); \ + } \ } \ } \ } \ diff --git a/vm_insnhelper.c b/vm_insnhelper.c index ba7523eb285d07..2c9bb027528b13 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -405,7 +405,7 @@ vm_push_frame(rb_execution_context_t *ec, *cfp = (const struct rb_control_frame_struct) { .pc = pc, .sp = sp, - .iseq = iseq, + ._iseq = iseq, .self = self, .ep = sp - 1, .block_code = NULL, @@ -1767,16 +1767,16 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c escape_cfp = reg_cfp; while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) { - if (ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) { escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp); ep = escape_cfp->ep; - base_iseq = escape_cfp->iseq; + base_iseq = CFP_ISEQ(escape_cfp); } else { ep = VM_ENV_PREV_EP(ep); base_iseq = ISEQ_BODY(base_iseq)->parent_iseq; escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep); - VM_ASSERT(escape_cfp->iseq == base_iseq); + VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq); } } @@ -1790,8 +1790,8 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c while (escape_cfp < eocfp) { if (escape_cfp->ep == ep) { - const rb_iseq_t *const iseq = escape_cfp->iseq; - const VALUE epc = escape_cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp); + const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table; unsigned int i; @@ -1850,7 +1850,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c if (lep == target_lep && VM_FRAME_RUBYFRAME_P(escape_cfp) && - ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) { in_class_frame = 1; target_lep = 0; } @@ -1880,7 +1880,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) { - switch (ISEQ_BODY(escape_cfp->iseq)->type) { + switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) { case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: if (toplevel) { @@ -1894,7 +1894,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } break; case ISEQ_TYPE_EVAL: { - const rb_iseq_t *is = escape_cfp->iseq; + const rb_iseq_t *is = CFP_ISEQ(escape_cfp); enum rb_iseq_type t = ISEQ_BODY(is)->type; while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) { if (!(is = ISEQ_BODY(is)->parent_iseq)) break; @@ -1912,7 +1912,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } - if (escape_cfp->ep == target_lep && ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_METHOD) { + if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) { if (target_ep == NULL) { goto valid_return; } @@ -2376,7 +2376,7 @@ vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct } #endif - return vm_search_method_slowpath0((VALUE)reg_cfp->iseq, cd, klass); + return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass); } static const struct rb_callable_method_entry_struct * @@ -2661,18 +2661,18 @@ vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - if (cfp->iseq && VM_FRAME_RUBYFRAME_P(cfp)) { - VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE; + if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) { + VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int lts = ISEQ_BODY(cfp->iseq)->local_table_size; - int params = ISEQ_BODY(cfp->iseq)->param.size; + if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size; + int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size; CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local bp += vm_ci_argc(ci); } - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { /* adjust `self' */ bp += 1; } @@ -3406,7 +3406,7 @@ vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct iseq = env->iseq; } else { // Otherwise use the lep to find the caller - iseq = rb_vm_search_cf_from_ep(ec, cfp, lep)->iseq; + iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep)); } // Our local storage is below the args we need to copy @@ -4613,8 +4613,8 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) { rb_control_frame_t *top_cfp = cfp; - if (cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_BLOCK) { - const rb_iseq_t *local_iseq = ISEQ_BODY(cfp->iseq)->local_iseq; + if (CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) { + const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq; do { cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -4622,7 +4622,7 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) /* TODO: orphan block */ return top_cfp; } - } while (cfp->iseq != local_iseq); + } while (CFP_ISEQ(cfp) != local_iseq); } return cfp; } @@ -4705,7 +4705,7 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c if (ref_cme) { if (calling->cd->cc) { const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); - RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc); + RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc); return vm_call_method(ec, cfp, calling); } else { @@ -5126,6 +5126,7 @@ static const struct rb_callcache * vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE current_defined_class; + const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp); const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp); if (!me) { @@ -5135,7 +5136,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c current_defined_class = vm_defined_class_for_protected_call(me); if (BUILTIN_TYPE(current_defined_class) != T_MODULE && - reg_cfp->iseq != method_entry_iseqptr(me) && + iseq != method_entry_iseqptr(me) && !rb_obj_is_kind_of(recv, current_defined_class)) { VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ? RCLASS_INCLUDER(current_defined_class) : current_defined_class; @@ -5167,7 +5168,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c vm_ci_argc(cd->ci), vm_ci_kwarg(cd->ci)); - RB_OBJ_WRITTEN(reg_cfp->iseq, Qundef, cd->ci); + RB_OBJ_WRITTEN(iseq, Qundef, cd->ci); } const struct rb_callcache *cc; @@ -5177,7 +5178,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c if (!klass) { /* bound instance method of module */ cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { cc = vm_search_method_fastpath(reg_cfp, cd, klass); @@ -5192,7 +5193,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid); if (cme) { cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { cd->cc = cc = empty_cc_for_super(); @@ -6155,6 +6156,7 @@ rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6168,7 +6170,7 @@ rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6197,6 +6199,7 @@ rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6209,7 +6212,7 @@ rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6617,7 +6620,7 @@ rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const vm_ic_track_const_chain(GET_CFP(), ic, segments); // Undo the PC increment to get the address to this instruction // INSN_ATTR(width) == 2 - vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2); + vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2); } return val; } @@ -6642,7 +6645,7 @@ vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is) RB_OBJ_SET_SHAREABLE(val); } - RB_OBJ_WRITE(ec->cfp->iseq, &is->once.value, val); + RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val); /* is->once.running_thread is cleared by vm_once_clear() */ is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */ @@ -6711,7 +6714,7 @@ vm_stack_consistency_error(const rb_execution_context_t *ec, #if defined RUBY_DEVEL VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp); rb_str_cat_cstr(mesg, "\n"); - rb_str_append(mesg, rb_iseq_disasm(cfp->iseq)); + rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp))); rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg)); #else rb_bug(stack_consistency_error, nsp, nbp); @@ -7241,7 +7244,7 @@ static VALUE rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp) { VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp)); - VM_ASSERT(ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE); + VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE); return cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } @@ -7257,7 +7260,7 @@ vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp) return; } else { - const rb_iseq_t *iseq = reg_cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp); size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded; rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos); unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; @@ -7543,7 +7546,7 @@ lookup_builtin_invoker(int argc) static inline VALUE invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv) { - const bool canary_p = ISEQ_BODY(reg_cfp->iseq)->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` + const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` SETUP_CANARY(canary_p); rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr; VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr); @@ -7563,7 +7566,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, if (0) { // debug print fputs("vm_invoke_builtin_delegate: passing -> ", stderr); for (int i=0; iargc; i++) { - ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(cfp->iseq)->local_table[i+start_index])); + ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index])); } ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc, (void *)(uintptr_t)bf->func_ptr); @@ -7573,7 +7576,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, return invoke_bf(ec, cfp, bf, NULL); } else { - const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; + const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; return invoke_bf(ec, cfp, bf, argv); } } diff --git a/vm_insnhelper.h b/vm_insnhelper.h index 6d32a7535b3308..88c387ee152afa 100644 --- a/vm_insnhelper.h +++ b/vm_insnhelper.h @@ -113,8 +113,11 @@ enum vm_regan_acttype { #define SET_SV(x) (*GET_SP() = rb_ractor_confirm_belonging(x)) /* set current stack value as x */ -/* instruction sequence C struct */ -#define GET_ISEQ() (GET_CFP()->iseq) +// instruction sequence C struct +// Uses cfp->_iseq directly because the interpreter always has a valid _iseq +// field (it's written on exit from JIT code). Code in vm_insnhelper.c that +// may be called as a ZJIT fallback should use rb_cfp_iseq() instead. +#define GET_ISEQ() (GET_CFP()->_iseq) /**********************************************************/ /* deal with variables */ diff --git a/vm_method.c b/vm_method.c index 521595d0b19666..021b06bf00109b 100644 --- a/vm_method.c +++ b/vm_method.c @@ -1084,7 +1084,7 @@ rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *de cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp && (line = rb_vm_get_sourceline(cfp))) { - VALUE location = rb_ary_new3(2, rb_iseq_path(cfp->iseq), INT2FIX(line)); + VALUE location = rb_ary_new3(2, rb_iseq_path(CFP_ISEQ(cfp)), INT2FIX(line)); rb_ary_freeze(location); RB_OBJ_SET_SHAREABLE(location); RB_OBJ_WRITE(me, &def->body.attr.location, location); @@ -2330,7 +2330,7 @@ scope_visibility_check(void) { /* Check for public/protected/private/module_function called inside a method */ rb_control_frame_t *cfp = GET_EC()->cfp+1; - if (cfp && cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD) { + if (cfp && CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD) { rb_warn("calling %s without arguments inside a method may not have the intended effect", rb_id2name(rb_frame_this_func())); } diff --git a/vm_trace.c b/vm_trace.c index 1dc176081440b0..74bc1fce807b83 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -812,7 +812,7 @@ get_path_and_lineno(const rb_execution_context_t *ec, const rb_control_frame_t * cfp = rb_vm_get_ruby_level_next_cfp(ec, cfp); if (cfp) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = CFP_ISEQ(cfp); *pathp = rb_iseq_path(iseq); if (event & (RUBY_EVENT_CLASS | @@ -862,7 +862,7 @@ call_trace_func(rb_event_flag_t event, VALUE proc, VALUE self, ID id, VALUE klas if (self && (filename != Qnil) && event != RUBY_EVENT_C_CALL && event != RUBY_EVENT_C_RETURN && - (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)ec->cfp->iseq, imemo_iseq))) { + (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)CFP_ISEQ(ec->cfp), imemo_iseq))) { argv[4] = rb_binding_new(); } argv[5] = klass ? klass : Qnil; @@ -1041,7 +1041,7 @@ rb_tracearg_parameters(rb_trace_arg_t *trace_arg) if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_BLOCK && !VM_FRAME_LAMBDA_P(cfp)) { is_proc = 1; } - return rb_iseq_parameters(cfp->iseq, is_proc); + return rb_iseq_parameters(CFP_ISEQ(cfp), is_proc); } break; } @@ -1103,7 +1103,7 @@ rb_tracearg_binding(rb_trace_arg_t *trace_arg) } cfp = rb_vm_get_binding_creatable_next_cfp(trace_arg->ec, trace_arg->cfp); - if (cfp && imemo_type_p((VALUE)cfp->iseq, imemo_iseq)) { + if (cfp && imemo_type_p((VALUE)CFP_ISEQ(cfp), imemo_iseq)) { return rb_vm_make_binding(trace_arg->ec, cfp); } else { diff --git a/yjit.c b/yjit.c index 2e7216a1915406..46565cb6c0d035 100644 --- a/yjit.c +++ b/yjit.c @@ -480,7 +480,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // If it's a FINISH frame, just normally exit with a non-Qundef value. cfp->jit_return = leave_exit; } - else if (cfp->jit_return) { + else if (CFP_JIT_RETURN(cfp)) { while (!VM_FRAME_FINISHED_P(cfp)) { if (cfp->jit_return == leave_exit) { // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on diff --git a/zjit.h b/zjit.h index d1d1b01df3b68e..e96caa257c6b91 100644 --- a/zjit.h +++ b/zjit.h @@ -9,6 +9,22 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif +// JITFrame is defined here as the single source of truth and imported into +// Rust via bindgen. C code reads fields directly; Rust uses an impl block. +typedef struct zjit_jit_frame { + // Program counter for this frame, used for backtraces and GC. + // NULL for C frames (they don't have a Ruby PC). + const VALUE *pc; + // The ISEQ this frame belongs to. Marked via rb_execution_context_mark. + // NULL for C frames. + const rb_iseq_t *iseq; + // Whether to materialize block_code when this frame is materialized. + // True when the ISEQ doesn't contain send/invokesuper/invokeblock + // (which write block_code themselves), so we must restore it. + // Always false for C frames. + bool materialize_block_code; +} zjit_jit_frame_t; + #if USE_ZJIT extern void *rb_zjit_entry; extern uint64_t rb_zjit_call_threshold; @@ -29,6 +45,7 @@ void rb_zjit_before_ractor_spawn(void); void rb_zjit_tracing_invalidate_all(void); void rb_zjit_invalidate_no_singleton_class(VALUE klass); void rb_zjit_invalidate_root_box(void); +void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); #else #define rb_zjit_entry 0 static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception) {} @@ -42,8 +59,45 @@ static inline void rb_zjit_before_ractor_spawn(void) {} static inline void rb_zjit_tracing_invalidate_all(void) {} static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {} static inline void rb_zjit_invalidate_root_box(void) {} +static inline void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame) {} #endif // #if USE_ZJIT #define rb_zjit_enabled_p (rb_zjit_entry != 0) +enum zjit_poison_values { + // Poison value used on frame push when runtime checks are enabled + ZJIT_JIT_RETURN_POISON = 2, +}; + +// Return the JITFrame pointer from cfp->jit_return, or NULL if not present. +// YJIT also uses jit_return (as a return address), so this must only return +// non-NULL when ZJIT is enabled and has set jit_return to a JITFrame pointer. +static inline void * +CFP_JIT_RETURN(const rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return NULL; +#if USE_ZJIT + RUBY_ASSERT_ALWAYS(cfp->jit_return != (void *)ZJIT_JIT_RETURN_POISON); +#endif + return cfp->jit_return; +} + +static inline const VALUE* +CFP_PC(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; + } + return cfp->pc; +} + +static inline const rb_iseq_t* +CFP_ISEQ(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; + } + return cfp->_iseq; +} + #endif // #ifndef ZJIT_H diff --git a/zjit.rb b/zjit.rb index e298b509049039..ffee0006962fc9 100644 --- a/zjit.rb +++ b/zjit.rb @@ -141,6 +141,7 @@ def stats_string ], buf:, stats:, right_align: true, base: :send_count) print_counters([ :compiled_iseq_count, + :compiled_side_exit_count, :failed_iseq_count, :compile_time_ns, diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 34720e77eb7ebe..b3f390026bc144 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -69,6 +69,9 @@ fn main() { // Import YARV bytecode instruction constants .allowlist_type("ruby_vminsn_type") + // JITFrame struct defined in zjit.h, imported into Rust via bindgen + .allowlist_type("zjit_jit_frame") + .allowlist_type("ruby_special_consts") .allowlist_function("rb_utf8_str_new") .allowlist_function("rb_str_buf_append") @@ -256,8 +259,7 @@ fn main() { .allowlist_type("iseq_inline_cvar_cache_entry") .blocklist_type("rb_execution_context_.*") // Large struct with various-type fields and an ifdef, so we don't import .opaque_type("rb_execution_context_.*") - .blocklist_type("rb_control_frame_struct") - .opaque_type("rb_control_frame_struct") + .allowlist_type("rb_control_frame_struct") .allowlist_function("rb_vm_bh_to_procval") .allowlist_function("rb_vm_env_write") .allowlist_function("rb_vm_ep_local_ep") @@ -277,6 +279,7 @@ fn main() { .allowlist_function("rb_iseq_(get|set)_zjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") + .allowlist_function("rb_iseq_bare_opcode_at_pc") .allowlist_function("rb_jit_reserve_addr_space") .allowlist_function("rb_jit_mark_writable") .allowlist_function("rb_jit_mark_executable") @@ -310,6 +313,7 @@ fn main() { .allowlist_function("rb_zjit_insn_leaf") .allowlist_type("jit_bindgen_constants") .allowlist_type("zjit_struct_offsets") + .allowlist_type("zjit_poison_values") .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_too_complex_p") .allowlist_function("rb_jit_multi_ractor_p") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 9083824c4b28af..ed126cdf284efa 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1754,7 +1754,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 76b11f26b2613b..624fcd0389f6f2 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use crate::bitset::BitSet; use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; -use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::cruby::{IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -551,6 +551,7 @@ pub struct SideExit { pub pc: Opnd, pub stack: Vec, pub locals: Vec, + pub iseq: IseqPtr, /// If set, the side exit will call the recompile function with these arguments /// to profile the send and invalidate the ISEQ for recompilation. pub recompile: Option, @@ -2619,7 +2620,7 @@ impl Assembler pub fn compile_exits(&mut self) -> Vec { /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { - let SideExit { pc, stack, locals, .. } = exit; + let SideExit { pc, stack, locals, iseq, .. } = exit; // Side exit blocks are not part of the CFG at the moment, // so we need to manually ensure that patchpoints get padded @@ -2632,6 +2633,11 @@ impl Assembler asm_comment!(asm, "save cfp->sp"); asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + asm_comment!(asm, "save cfp->iseq"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); + + // cfp->block_code and cfp->jit_return are cleared by the caller jit_exec() or JIT_EXEC() + if !stack.is_empty() { asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); for (idx, &opnd) in stack.iter().enumerate() { @@ -2663,6 +2669,16 @@ impl Assembler // compile_exit_save_state because it clobbers caller-saved registers // that may hold stack/local operands we need to save. if let Some(recompile) = &exit.recompile { + if cfg!(feature = "runtime_checks") { + // Clear jit_return to fully materialize the frame. This must happen + // before any C call in the exit path (e.g. no_profile_send_recompile) + // because that C call can trigger GC, which walks the stack and would + // hit the CFP_JIT_RETURN assertion if jit_return still holds the + // runtime_checks poison value (JIT_RETURN_POISON). + asm_comment!(asm, "clear cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + } + use crate::codegen::no_profile_send_recompile; asm_comment!(asm, "profile and maybe recompile for no-profile send"); asm_ccall!(asm, no_profile_send_recompile, @@ -2791,6 +2807,7 @@ impl Assembler if !compiled_exits.is_empty() { let nanos = side_exit_start.elapsed().as_nanos(); crate::stats::incr_counter_by(crate::stats::Counter::compile_side_exit_time_ns, nanos as u64); + crate::stats::incr_counter_by(crate::stats::Counter::compiled_side_exit_count, compiled_exits.len() as u64); } // Close the current perf range for side exits diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 3cf744990d8381..f7900f025ee0a1 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1388,7 +1388,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 9c4a456af91d2f..9846a0e70a23f1 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -14,7 +14,7 @@ use crate::invariants::{ track_root_box_assumption }; use crate::gc::append_gc_offsets; -use crate::payload::{get_or_create_iseq_payload, IseqCodePtrs, IseqVersion, IseqVersionRef, IseqStatus}; +use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; @@ -36,6 +36,13 @@ const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { None }; +/// Sentinel jit_return stored on ISEQ frame push when runtime checks are enabled. +const JIT_RETURN_POISON: Option = if cfg!(feature = "runtime_checks") { + Some(ZJIT_JIT_RETURN_POISON as usize) +} else { + None +}; + /// Ephemeral code generation state struct JITState { /// Instruction sequence for the method being compiled @@ -623,6 +630,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), // Ensure we have enough room fit ec, self, and arguments // TODO remove this check when we have stack args (we can use Time.new to test it) @@ -643,6 +651,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + Insn::IntAnd { left, right } => asm.and(opnd!(left), opnd!(right)), Insn::IntOr { left, right } => gen_int_or(asm, opnd!(left), opnd!(right)), &Insn::FixnumLShift { left, right, state } => { // We only create FixnumLShift when we know the shift amount statically and it's in [0, @@ -1043,8 +1052,8 @@ fn gen_ccall_with_frame( iseq: None, cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, - pc: PC_POISON, specval: block_handler_specval, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1133,7 +1142,7 @@ fn gen_ccall_variadic( cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, specval: block_handler_specval, - pc: PC_POISON, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1589,8 +1598,8 @@ fn gen_send_iseq_direct( iseq: Some(iseq), cme, frame_type, - pc: None, specval, + write_block_code: iseq_may_write_block_code(iseq), }); // Write "keyword_bits" to the callee's frame if the callee accepts keywords. @@ -1703,6 +1712,42 @@ fn gen_invokeblock( ) } +/// Compile invokeblock for IFUNC block handlers. +/// Calls rb_vm_yield_with_cfunc directly. +fn gen_invokeblock_ifunc( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + block_handler: Opnd, + args: Vec, + state: &FrameState, +) -> lir::Opnd { + let _ = cd; // cd is not needed for the direct call + + gen_prepare_non_leaf_call(jit, asm, state); + + // Push args to memory so we can pass argv pointer + let argv_ptr = gen_push_opnds(asm, &args); + + // Untag the block handler to get the captured block pointer + // captured = block_handler & ~0x3 + asm_comment!(asm, "untag block handler to get captured block"); + let captured = asm.and(block_handler, Opnd::Imm(!0x3i64)); + + asm_comment!(asm, "call rb_vm_yield_with_cfunc"); + unsafe extern "C" { + fn rb_vm_yield_with_cfunc( + ec: EcPtr, + captured: VALUE, + argc: i32, + argv: *const VALUE, + ) -> VALUE; + } + let result = asm_ccall!(asm, rb_vm_yield_with_cfunc, EC, captured, (args.len() as i64).into(), argv_ptr); + gen_pop_opnds(asm, &args); + result +} + fn gen_invokeproc( jit: &mut JITState, asm: &mut Assembler, @@ -2658,6 +2703,33 @@ fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReaso } } +/// Check if an ISEQ contains instructions that may write to block_code +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock). +/// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. +#[allow(non_upper_case_globals)] +fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { + let encoded_size = unsafe { rb_iseq_encoded_size(iseq) }; + let mut insn_idx: u32 = 0; + + while insn_idx < encoded_size { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let opcode = unsafe { rb_iseq_bare_opcode_at_pc(iseq, pc) } as u32; + + match opcode { + YARVINSN_send | YARVINSN_sendforward | + YARVINSN_invokesuper | YARVINSN_invokesuperforward | + YARVINSN_invokeblock => { + return true; + } + _ => {} + } + + insn_idx = insn_idx.saturating_add(unsafe { rb_insn_len(VALUE(opcode as usize)) }.try_into().unwrap()); + } + + false +} + /// Save only the PC to CFP. Use this when you need to call gen_save_sp() /// immediately after with a custom stack size (e.g., gen_ccall_with_frame /// adjusts SP to exclude receiver and arguments). @@ -2667,7 +2739,11 @@ fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { gen_incr_counter(asm, Counter::vm_write_pc_count); asm_comment!(asm, "save PC to CFP"); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); + if let Some(pc) = PC_POISON { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } /// Save the current PC on the CFP as a preparation for calling a C function @@ -2762,7 +2838,9 @@ struct ControlFrame { /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. /// For the type of frames we push, block handler or the parent EP. specval: lir::Opnd, - pc: Option<*const VALUE>, + /// Whether to write block_code = 0 at frame push time. + /// True when the callee ISEQ may write to block_code (has send/invokesuper/invokeblock). + write_block_code: bool, } /// Compile an interpreter frame @@ -2792,25 +2870,35 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C asm_comment!(asm, "push callee control frame"); - if let Some(iseq) = frame.iseq { - // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits, non-leaf calls, or calls with GC - // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits, non-leaf calls, or calls with GC - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + if frame.iseq.is_some() { + // PC, SP, and ISEQ are written lazily by the callee on side-exits, non-leaf calls, or GC. + if let Some(jit_return_poison) = JIT_RETURN_POISON { + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), jit_return_poison.into()); + } + if frame.write_block_code { + asm_comment!(asm, "write block_code for iseq that may use it"); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + } } else { - // C frames don't have a PC and ISEQ in normal operation. - // When runtime checks are enabled we poison the PC so accidental reads stand out. - if let Some(pc) = frame.pc { - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + // C frames don't have a PC and ISEQ in normal operation. ISEQ frames set PC on gen_save_pc_for_gc(). + // When runtime checks are enabled we poison the PC for C frames so accidental reads stand out. + if let (None, Some(pc)) = (frame.iseq, PC_POISON) { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); } let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); + // block_code must be written explicitly because the interpreter reads + // captured->code.ifunc directly from cfp->block_code (not through JITFrame). + // Without this, stale data from a previous frame occupying this CFP slot + // can be used as an ifunc pointer, causing a segfault. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + let jit_frame = JITFrame::new_cfunc(); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); } /// Stack overflow check: fails if CFP<=SP at any point in the callee. @@ -2894,6 +2982,7 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { pc: Opnd::const_ptr(state.pc), stack, locals, + iseq: jit.iseq, recompile: None, } } @@ -2971,20 +3060,35 @@ c_callable! { /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. - fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { - with_vm_lock(src_loc!(), || { - // gen_push_frame() doesn't set PC, so we need to set them before exit. - // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE, ec: EcPtr) -> *const u8 { + // Make sure cfp is ready to be scanned by other Ractors and GC before taking the barrier + { + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; let iseq = iseq_call.iseq.get(); let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + // gen_push_frame() doesn't set PC or ISEQ, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC and ISEQ. + // Clear jit_return so the interpreter reads cfp->pc and cfp->iseq directly. let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; + unsafe { (*cfp)._iseq = iseq }; + unsafe { (*cfp).jit_return = std::ptr::null_mut() }; + } + + with_vm_lock(src_loc!(), || { + // Re-create the Rc inside the VM lock because IseqCall's interior + // mutability (Cell) requires exclusive access. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); // JIT-to-JIT calls don't eagerly fill nils to non-parameter locals. // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { unsafe { + // Caller frames are materialized by jit_exec() after the entry trampoline returns. + // The current frame's pc and iseq are already set by function_stub_hit before this point. + // Set SP which gen_push_frame() doesn't set rb_set_cfp_sp(cfp, sp); @@ -3001,15 +3105,21 @@ c_callable! { } } - // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). + // If we already know we can't compile the ISEQ, or there is insufficient native + // stack space, fail early without cb.mark_all_executable(). // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. let cb = ZJITState::get_code_block(); + let native_stack_full = unsafe { rb_ec_stack_check(ec as _) } != 0; let payload = get_or_create_iseq_payload(iseq); let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); let compile_error = match last_status { Some(IseqStatus::CantCompile(err)) => Some(err), _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ if native_stack_full => { + incr_counter!(skipped_native_stack_full); + Some(&CompileError::NativeStackTooLarge) + }, _ => None, }; if let Some(compile_error) = compile_error { @@ -3114,7 +3224,7 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result, } -/// Opaque control_frame (CFP) struct from vm_core.h -#[repr(C)] -pub struct rb_control_frame_struct { - _data: [u8; 0], - _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, -} - /// Pointer to a control frame pointer (CFP) pub type CfpPtr = *mut rb_control_frame_struct; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 5c7ce49fc612da..d8fdcbb84538c3 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1299,6 +1299,16 @@ pub struct rb_block__bindgen_ty_1 { pub proc_: __BindgenUnionField, pub bindgen_union_field: [u64; 3usize], } +#[repr(C)] +pub struct rb_control_frame_struct { + pub pc: *const VALUE, + pub sp: *mut VALUE, + pub _iseq: *const rb_iseq_t, + pub self_: VALUE, + pub ep: *const VALUE, + pub block_code: *const ::std::os::raw::c_void, + pub jit_return: *mut ::std::os::raw::c_void, +} pub type rb_control_frame_t = rb_control_frame_struct; #[repr(C)] pub struct rb_proc_t { @@ -1900,6 +1910,15 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct zjit_jit_frame { + pub pc: *const VALUE, + pub iseq: *const rb_iseq_t, + pub materialize_block_code: bool, +} +pub const ZJIT_JIT_RETURN_POISON: zjit_poison_values = 2; +pub type zjit_poison_values = u32; pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; pub type zjit_struct_offsets = u32; pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; @@ -2169,6 +2188,10 @@ unsafe extern "C" { pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; + pub fn rb_iseq_bare_opcode_at_pc( + iseq: *const rb_iseq_t, + pc: *const VALUE, + ) -> ::std::os::raw::c_int; pub fn rb_RSTRING_LEN(str_: VALUE) -> ::std::os::raw::c_ulong; pub fn rb_RSTRING_PTR(str_: VALUE) -> *mut ::std::os::raw::c_char; pub fn rb_insn_name(insn: VALUE) -> *const ::std::os::raw::c_char; diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index b79f19837ff09a..239b71d5f48754 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -89,6 +89,14 @@ pub extern "C" fn rb_zjit_root_update_references() { } let invariants = ZJITState::get_invariants(); invariants.update_references(); + + // Update iseq pointers in all JITFrames for GC compaction. + // rb_execution_context_update only updates JITFrames currently on the stack, + // but JITFrames not on the stack also need their iseq pointers updated + // because the JIT code will reuse them on the next call. + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &mut *jit_frame }.update_references(); + } } fn iseq_mark(payload: &IseqPayload) { @@ -206,5 +214,13 @@ fn ranges_overlap(left: &Range, right: &Range) -> bool where T: Partial /// Callback for marking GC objects inside [crate::invariants::Invariants]. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_root_mark() { - // TODO(max): Either add roots to mark or consider removing this callback + // Mark iseq pointers in all JITFrames. JITFrames that are currently on the + // stack are also marked via rb_execution_context_mark, but JITFrames not on + // the stack still need their iseqs kept alive because JIT code will reuse them. + if !ZJITState::has_instance() { + return; + } + for &jit_frame in ZJITState::get_jit_frames().iter() { + unsafe { &*jit_frame }.mark(); + } } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 7107f7046afbb9..b0a53ec56493f4 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -538,6 +538,7 @@ pub enum SideExitReason { DirectiveInduced, SendWhileTracing, NoProfileSend, + InvokeBlockNotIfunc, } #[derive(Debug, Clone, Copy)] @@ -698,6 +699,16 @@ pub enum SendFallbackReason { SuperPolymorphic, /// The `super` target call uses a complex argument pattern that the optimizer does not support. SuperTargetComplexArgsPass, + /// The `invokeblock` instruction is not yet optimized in `type_specialize`. + InvokeBlockNotSpecialized, + /// The `sendforward` instruction (argument forwarding `...`) is not yet optimized in + /// `type_specialize`. + SendForwardNotSpecialized, + /// The `invokesuperforward` instruction (super with forwarding `...`) is not yet optimized in + /// `type_specialize`. + InvokeSuperForwardNotSpecialized, + /// The single-ractor-mode assumption could not be made. + SingleRactorModeRequired, /// Initial fallback reason for every instruction, which should be mutated to /// a more actionable reason when an attempt to specialize the instruction fails. Uncategorized(ruby_vminsn_type), @@ -745,6 +756,10 @@ impl Display for SendFallbackReason { SuperPolymorphic => write!(f, "super: polymorphic call site"), SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + InvokeBlockNotSpecialized => write!(f, "InvokeBlock: not yet specialized"), + SendForwardNotSpecialized => write!(f, "SendForward: not yet specialized"), + InvokeSuperForwardNotSpecialized => write!(f, "InvokeSuperForward: not yet specialized"), + SingleRactorModeRequired => write!(f, "Single-ractor mode required"), Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), } } @@ -997,6 +1012,14 @@ pub enum Insn { state: InsnId, reason: SendFallbackReason, }, + /// Optimized invokeblock for IFUNC block handlers. + /// Calls rb_vm_yield_with_cfunc directly instead of going through rb_vm_invokeblock. + InvokeBlockIfunc { + cd: *const rb_call_data, + block_handler: InsnId, + args: Vec, + state: InsnId, + }, /// Call Proc#call optimized method type. InvokeProc { recv: InsnId, @@ -1049,6 +1072,7 @@ pub enum Insn { FixnumAnd { left: InsnId, right: InsnId }, FixnumOr { left: InsnId, right: InsnId }, FixnumXor { left: InsnId, right: InsnId }, + IntAnd { left: InsnId, right: InsnId }, IntOr { left: InsnId, right: InsnId }, FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, FixnumRShift { left: InsnId, right: InsnId }, @@ -1252,6 +1276,7 @@ macro_rules! for_each_operand_impl { | Insn::FixnumAnd { left, right } | Insn::FixnumOr { left, right } | Insn::FixnumXor { left, right } + | Insn::IntAnd { left, right } | Insn::IntOr { left, right } | Insn::FixnumRShift { left, right } | Insn::IsBitEqual { left, right } @@ -1321,6 +1346,11 @@ macro_rules! for_each_operand_impl { $visit_many!(args); $visit_one!(state); } + Insn::InvokeBlockIfunc { block_handler, args, state, .. } => { + $visit_one!(block_handler); + $visit_many!(args); + $visit_one!(state); + } Insn::CCall { recv, args, .. } => { $visit_one!(recv); $visit_many!(args); @@ -1569,6 +1599,7 @@ impl Insn { Insn::InvokeSuper { .. } => effects::Any, Insn::InvokeSuperForward { .. } => effects::Any, Insn::InvokeBlock { .. } => effects::Any, + Insn::InvokeBlockIfunc { .. } => effects::Any, Insn::SendDirect { .. } => effects::Any, Insn::InvokeBuiltin { .. } => effects::Any, Insn::EntryPoint { .. } => effects::Any, @@ -1588,6 +1619,7 @@ impl Insn { Insn::FixnumAnd { .. } => effects::Empty, Insn::FixnumOr { .. } => effects::Empty, Insn::FixnumXor { .. } => effects::Empty, + Insn::IntAnd { .. } => effects::Empty, Insn::IntOr { .. } => effects::Empty, Insn::FixnumLShift { .. } => effects::Empty, Insn::FixnumRShift { .. } => effects::Empty, @@ -1932,6 +1964,13 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } + Insn::InvokeBlockIfunc { block_handler, args, .. } => { + write!(f, "InvokeBlockIfunc {block_handler}")?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } Insn::InvokeProc { recv, args, kw_splat, .. } => { write!(f, "InvokeProc {recv}")?; for arg in args { @@ -1970,6 +2009,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::IntAnd { left, right } => { write!(f, "IntAnd {left}, {right}") }, Insn::IntOr { left, right } => { write!(f, "IntOr {left}, {right}") }, Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, @@ -2759,6 +2799,7 @@ impl Function { &FixnumAnd { left, right } => FixnumAnd { left: find!(left), right: find!(right) }, &FixnumOr { left, right } => FixnumOr { left: find!(left), right: find!(right) }, &FixnumXor { left, right } => FixnumXor { left: find!(left), right: find!(right) }, + &IntAnd { left, right } => IntAnd { left: find!(left), right: find!(right) }, &IntOr { left, right } => IntOr { left: find!(left), right: find!(right) }, &FixnumLShift { left, right, state } => FixnumLShift { left: find!(left), right: find!(right), state }, &FixnumRShift { left, right } => FixnumRShift { left: find!(left), right: find!(right) }, @@ -2820,6 +2861,12 @@ impl Function { state, reason, }, + &InvokeBlockIfunc { cd, block_handler, ref args, state } => InvokeBlockIfunc { + cd, + block_handler: find!(block_handler), + args: find_vec!(args), + state: find!(state), + }, &InvokeProc { recv, ref args, state, kw_splat } => InvokeProc { recv: find!(recv), args: find_vec!(args), @@ -3015,6 +3062,7 @@ impl Function { Insn::FixnumAnd { .. } => types::Fixnum, Insn::FixnumOr { .. } => types::Fixnum, Insn::FixnumXor { .. } => types::Fixnum, + Insn::IntAnd { .. } => types::CInt64, Insn::IntOr { left, .. } => self.type_of(*left).unspecialized(), Insn::FixnumLShift { .. } => types::Fixnum, Insn::FixnumRShift { .. } => types::Fixnum, @@ -3025,6 +3073,7 @@ impl Function { Insn::InvokeSuper { .. } => types::BasicObject, Insn::InvokeSuperForward { .. } => types::BasicObject, Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeBlockIfunc { .. } => types::BasicObject, Insn::InvokeProc { .. } => types::BasicObject, Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), @@ -3716,6 +3765,7 @@ impl Function { // Check for "defined with an un-shareable Proc in a different Ractor" if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } // Check singleton class assumption first, before emitting other patchpoints @@ -3735,6 +3785,7 @@ impl Function { // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } // Check singleton class assumption first, before emitting other patchpoints @@ -3755,6 +3806,7 @@ impl Function { // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } @@ -3802,12 +3854,15 @@ impl Function { { let native_index = (index as i64) * (SIZEOF_VALUE as i64); if native_index > (i32::MAX as i64) { + self.set_dynamic_send_reason(insn_id, TooManyArgsForLir); self.push_insn_id(block, insn_id); continue; } } // Get the profiled type to check if the fields is embedded or heap allocated. let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { // No (monomorphic/skewed polymorphic) profile info + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); self.push_insn_id(block, insn_id); continue; }; // Check singleton class assumption first, before emitting other patchpoints @@ -6109,6 +6164,7 @@ impl Function { } // Instructions with a Vec of Ruby objects Insn::InvokeBlock { ref args, .. } + | Insn::InvokeBlockIfunc { ref args, .. } | Insn::NewArray { elements: ref args, .. } | Insn::ArrayHash { elements: ref args, .. } | Insn::ArrayMin { elements: ref args, .. } @@ -6198,7 +6254,8 @@ impl Function { Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())) } } - Insn::IntOr { left, right } => { + Insn::IntAnd { left, right } + | Insn::IntOr { left, right } => { // TODO: Expand this to other matching C integer sizes when we need them. self.assert_subtype(insn_id, left, types::CInt64)?; self.assert_subtype(insn_id, right, types::CInt64) @@ -6463,7 +6520,7 @@ impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { #[derive(Debug, Clone, PartialEq)] pub struct FrameState { - iseq: IseqPtr, + pub iseq: IseqPtr, insn_idx: usize, // Ruby bytecode instruction pointer pub pc: *const VALUE, @@ -7984,7 +8041,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; let recv = state.stack_pop()?; - let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: SendForwardNotSpecialized }); state.stack_push(send_forward); if !blockiseq.is_null() { @@ -8056,7 +8113,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let argc = unsafe { vm_ci_argc((*cd).ci) }; let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; let recv = state.stack_pop()?; - let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: InvokeSuperForwardNotSpecialized }); state.stack_push(result); if !blockiseq.is_null() { @@ -8091,7 +8148,46 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let argc = unsafe { vm_ci_argc((*cd).ci) }; let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; - let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: Uncategorized(opcode) }); + + // Check if this is a monomorphic IFUNC block handler we can specialize + let block_handler_types = profiles.payload.profile.get_operand_types(exit_state.insn_idx); + let is_ifunc = (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT)) == 0 + && block_handler_types.is_some_and(|types| types.len() == 1 && { + let summary = TypeDistributionSummary::new(&types[0]); + summary.is_monomorphic() && unsafe { rb_IMEMO_TYPE_P(summary.bucket(0).class(), imemo_ifunc) == 1 } + }); + + let result = if is_ifunc { + // Get the local EP to load the block handler + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + let block_handler = fun.push_insn(block, Insn::LoadField { + recv: lep, + id: ID!(_env_data_index_specval), + offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, + return_type: types::CInt64, + }); + + // Guard that the block handler is an IFUNC (tag bits & 0x3 == 0x3), + // matching VM_BH_IFUNC_P() in the interpreter. + let tag_mask = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let tag_bits = fun.push_insn(block, Insn::IntAnd { left: block_handler, right: tag_mask }); + fun.push_insn(block, Insn::GuardBitEquals { + val: tag_bits, + expected: Const::CInt64(0x3), + reason: SideExitReason::InvokeBlockNotIfunc, + state: exit_id, + }); + + fun.push_insn(block, Insn::InvokeBlockIfunc { + cd, + block_handler, + args, + state: exit_id, + }) + } else { + fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }) + }; state.stack_push(result); } YARVINSN_getglobal => { diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 3cf02934e7ad23..c9f2a45b048adc 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -7517,7 +7517,7 @@ mod hir_opt_tests { v4:BasicObject = LoadArg :self@0 Jump bb3(v4) bb3(v6:BasicObject): - v11:BasicObject = Send v6, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + v11:BasicObject = Send v6, :foo # SendFallbackReason: Single-ractor mode required CheckInterrupts Return v11 "); @@ -14810,7 +14810,7 @@ mod hir_opt_tests { v88:Array = RefineType v67, Array v89:CInt64 = UnboxFixnum v68 v90:BasicObject = ArrayAref v88, v89 - v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: Uncategorized(invokeblock) + v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: InvokeBlock: not yet specialized v91:Fixnum[1] = Const Value(1) v92:Fixnum = FixnumAdd v68, v91 PatchPoint NoEPEscape(each) @@ -15075,4 +15075,46 @@ mod hir_opt_tests { Return v36 "); } + + #[test] + fn test_invokeblock_ifunc() { + eval(" + class IFuncTestList + include Enumerable + def each + yield 1 + yield 2 + end + end + IFuncTestList.new.map { |x| x } + "); + assert_snapshot!(hir_string_proc("IFuncTestList.instance_method(:each)"), @" + fn each@:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:CPtr = GetEP 0 + v13:CInt64 = LoadField v12, :_env_data_index_specval@0x1000 + v14:CInt64[3] = Const CInt64(3) + v15:CInt64 = IntAnd v13, v14 + v16:CInt64[3] = GuardBitEquals v15, CInt64(3) + v17:BasicObject = InvokeBlockIfunc v13, v10 + v21:Fixnum[2] = Const Value(2) + v23:CPtr = GetEP 0 + v24:CInt64 = LoadField v23, :_env_data_index_specval@0x1000 + v25:CInt64[3] = Const CInt64(3) + v26:CInt64 = IntAnd v24, v25 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) + v28:BasicObject = InvokeBlockIfunc v24, v21 + CheckInterrupts + Return v28 + "); + } } diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 606df902f20183..72012b7eba9504 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -2240,7 +2240,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized CheckInterrupts Return v16 "); @@ -2265,7 +2265,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized v17:CPtr = GetEP 0 v18:BasicObject = LoadField v17, :...@0x1010 CheckInterrupts @@ -2292,7 +2292,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized v18:Fixnum[1] = Const Value(1) v21:BasicObject = Send v16, :+, v18 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts @@ -2320,7 +2320,7 @@ pub mod hir_build_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) - v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized CheckInterrupts Return v18 "); @@ -2431,7 +2431,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: Uncategorized(sendforward) + v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: SendForward: not yet specialized CheckInterrupts Return v16 "); @@ -4369,7 +4369,7 @@ pub mod hir_build_tests { v42 = RefineType v38, Falsy IfFalse v41, bb4(v18, v19, v20, v21, v22, v27) v44:ObjectSubclass[BlockParamProxy] = RefineType v38, Truthy - v48:BasicObject = InvokeBlock, v27 # SendFallbackReason: Uncategorized(invokeblock) + v48:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized v51:BasicObject = InvokeBuiltin dir_s_close, v18, v27 CheckInterrupts Return v48 @@ -4723,7 +4723,7 @@ pub mod hir_build_tests { v4:BasicObject = LoadArg :self@0 Jump bb3(v4) bb3(v6:BasicObject): - v10:BasicObject = InvokeBlock # SendFallbackReason: Uncategorized(invokeblock) + v10:BasicObject = InvokeBlock # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts Return v10 "); @@ -4752,7 +4752,7 @@ pub mod hir_build_tests { v9:BasicObject = LoadArg :y@2 Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): - v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: Uncategorized(invokeblock) + v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts Return v19 "); @@ -5038,7 +5038,7 @@ pub mod hir_build_tests { Return v48 bb7(v67:BasicObject, v68:Fixnum): v72:BasicObject = InvokeBuiltin rb_jit_ary_at, v67, v68 - v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: Uncategorized(invokeblock) + v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: InvokeBlock: not yet specialized v78:Fixnum = InvokeBuiltin rb_jit_fixnum_inc, v67, v68 PatchPoint NoEPEscape(each) Jump bb8(v67, v78) diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs new file mode 100644 index 00000000000000..f7133daab2ee79 --- /dev/null +++ b/zjit/src/jit_frame.rs @@ -0,0 +1,318 @@ +use crate::cruby::{IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; +use crate::cruby::zjit_jit_frame; +use crate::state::ZJITState; + +/// JITFrame struct is defined in zjit.h (the single source of truth) and +/// imported into Rust via bindgen. See zjit.h for field documentation. +pub type JITFrame = zjit_jit_frame; + +impl JITFrame { + /// Allocate a JITFrame on the heap, register it with ZJITState, and return + /// a raw pointer that remains valid for the lifetime of the process. + fn alloc(jit_frame: JITFrame) -> *const Self { + let raw_ptr = Box::into_raw(Box::new(jit_frame)); + ZJITState::get_jit_frames().push(raw_ptr); + raw_ptr as *const _ + } + + /// Create a JITFrame for an ISEQ frame. + pub fn new_iseq(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { + Self::alloc(JITFrame { pc, iseq, materialize_block_code }) + } + + /// Create a JITFrame for a C frame (no PC, no ISEQ). + pub fn new_cfunc() -> *const Self { + Self::alloc(JITFrame { pc: std::ptr::null(), iseq: std::ptr::null(), materialize_block_code: false }) + } + + /// Mark the iseq pointer for GC. Called from rb_zjit_root_mark. + pub fn mark(&self) { + if !self.iseq.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from(self.iseq)); } + } + } + + /// Update the iseq pointer after GC compaction. + pub fn update_references(&mut self) { + if !self.iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(self.iseq)) }.as_iseq(); + if self.iseq != new_iseq { + self.iseq = new_iseq; + } + } + } +} + +/// Update the iseq pointer in an on-stack JITFrame during GC compaction. +/// Called from rb_execution_context_update in vm.c. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_frame_update_references(jit_frame: *mut JITFrame) { + unsafe { &mut *jit_frame }.update_references(); +} + +#[cfg(test)] +mod tests { + use crate::cruby::{eval, inspect}; + use insta::assert_snapshot; + + #[test] + fn test_jit_frame_entry_first() { + eval(r#" + def test + itself + callee + end + + def callee + caller + end + + test + "#); + assert_snapshot!(inspect("test.first"), @r#"":4:in 'Object#test'""#); + } + + #[test] + fn test_materialize_one_frame() { + assert_snapshot!(inspect(" + def jit_entry + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + #[test] + fn test_materialize_two_frames() { // materialize caller frames on raise + // At the point of `resuce`, there are two lightweight frames on stack and both need to be + // materialized before passing control to interpreter. + assert_snapshot!(inspect(" + def jit_entry = raise_and_rescue + def raise_and_rescue + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + // Materialize frames on side exit: a type guard triggers a side exit with + // multiple JIT frames on the stack. All frames must be materialized before + // the interpreter resumes. + #[test] + fn test_side_exit_materialize_frames() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); + } + + // BOP invalidation must not overwrite the top-most frame's PC with + // jit_frame's PC. After invalidation the interpreter resumes at a new + // PC, so a stale jit_frame PC would cause wrong execution. + #[test] + fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); + } + + // Side exit at the very start of a method, before any jit_return has been + // written by gen_save_pc_for_gc. The jit_return field should be 0 (from + // vm_push_frame), so materialization should be a no-op for that frame. + #[test] + fn test_side_exit_before_jit_return_write() { + assert_snapshot!(inspect(" + def entry(n) = n + 1 + entry(1) + [entry(1), entry(1.0)] + "), @"[2, 2.0]"); + } + + #[test] + fn test_caller_iseq() { + assert_snapshot!(inspect(r#" + def callee = call_caller + def test = callee + + def callee2 = call_caller + def test2 = callee2 + + def call_caller = caller + + test + test2 + test.first + "#), @r#"":2:in 'Object#callee'""#); + } + + // ISEQ must be readable during exception handling so the interpreter + // can look up rescue/ensure tables. + #[test] + fn test_iseq_on_raise() { + assert_snapshot!(inspect(r#" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "#), @"(0/1)..1"); + } + + // Multiple exception raises during keyword argument evaluation: each + // raise needs correct ISEQ for catch table lookup. + #[test] + fn test_iseq_on_raise_on_ensure() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); + } + + // Send fallback (e.g. method_missing) calls into the interpreter, which + // reads cfp->iseq via GET_ISEQ(). gen_prepare_non_leaf_call writes the + // iseq to JITFrame, but GET_ISEQ reads cfp->iseq directly. This test + // ensures the interpreter can resolve the caller iseq for backtraces. + #[test] + fn test_send_fallback_caller_location() { + assert_snapshot!(inspect(r#" + def callee = caller_locations(1, 1)[0].label + def test = callee + test + test + "#), @r#""Object#test""#); + } + + // A send fallback may throw (e.g. via method_missing raising). The + // interpreter must be able to find the correct rescue handler in the + // caller's ISEQ catch table. This exercises throw through send fallback. + #[test] + fn test_send_fallback_throw() { + assert_snapshot!(inspect(r#" + class Foo + def method_missing(name, *) = raise("no #{name}") + end + def test + Foo.new.bar + rescue RuntimeError => e + e.message + end + test + test + "#), @r#""no bar""#); + } + + // Proc.new inside a block passed via invokeblock captures the caller's + // block_code. When the JIT compiles the caller, block_code must be + // correctly available for the proc to work. + #[test] + fn test_proc_from_invokeblock() { + assert_snapshot!(inspect(" + def capture_block(&blk) = blk + def test = capture_block { 42 } + test + test.call + "), @"42"); + } + + // binding() called from a JIT-compiled callee must see the correct + // source location (iseq + pc) of the caller frame. + #[test] + fn test_binding_source_location() { + assert_snapshot!(inspect(r#" + def callee = binding + def test = callee + test + b = test + b.source_location[1] > 0 + "#), @"true"); + } + + // $~ (Regexp special variable) is stored via svar which walks the EP + // chain to find the LEP. rb_vm_svar_lep uses rb_zjit_cfp_has_iseq to + // skip C frames, so it must work correctly with JITFrame. + #[test] + fn test_svar_regexp_match() { + assert_snapshot!(inspect(r#" + def test(s) + s =~ /hello/ + $~ + end + test("hello world") + test("hello world").to_s + "#), @r#""hello""#); + } + + // C function calls with rb_block_call (like Array#each, Enumerable#map) + // write an ifunc to cfp->block_code after the JIT pushes the C frame. + // GC must mark and relocate this ifunc. This test exercises the code + // path fixed by "Fix ZJIT segfault: write block_code for C frames and + // fix GC marking". + #[test] + fn test_cfunc_block_code_gc() { + assert_snapshot!(inspect(" + def test + # Use a cfunc that calls back into Ruby with a block (rb_block_call) + [1, 2, 3].map { |x| x.to_s } + end + test + test + "), @r#"["1", "2", "3"]"#); + } + + // Multiple levels of cfunc-with-block: a JIT-compiled method calls a + // cfunc that yields, and the block itself calls another cfunc that + // yields. Each C frame's block_code must be properly initialized. + #[test] + fn test_nested_cfunc_with_block() { + assert_snapshot!(inspect(" + def test + [1, 2].flat_map { |x| [x, x + 10].map { |y| y * 2 } } + end + test + test + "), @"[2, 22, 4, 24]"); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs index 50a48399c2c2d1..1440b6ff6942e6 100644 --- a/zjit/src/lib.rs +++ b/zjit/src/lib.rs @@ -29,6 +29,7 @@ mod profile; mod invariants; mod bitset; mod gc; +mod jit_frame; mod payload; mod json; mod ttycolors; diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs index 660e2a80ce9101..567c6e9eea10f8 100644 --- a/zjit/src/payload.rs +++ b/zjit/src/payload.rs @@ -4,6 +4,8 @@ use crate::codegen::IseqCallRef; use crate::stats::CompileError; use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; +pub use crate::jit_frame::JITFrame; + /// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. #[derive(Debug)] pub struct IseqPayload { diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 6c61bc77efd314..813e23dd4140c0 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -8,6 +8,7 @@ use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::jit_frame::JITFrame; use crate::stats::{Counters, InsnCounters, PerfettoTracer}; use crate::virtualmem::CodePtr; use std::sync::atomic::AtomicUsize; @@ -70,6 +71,9 @@ pub struct ZJITState { /// Perfetto tracer for --zjit-trace-exits perfetto_tracer: Option, + + /// Frame metadata for ISEQ and C calls that are known at compile time + jit_frames: Vec<*mut JITFrame>, } /// Tracks the initialization progress @@ -147,6 +151,7 @@ impl ZJITState { ccall_counter_pointers: HashMap::new(), iseq_calls_count_pointers: HashMap::new(), perfetto_tracer, + jit_frames: vec![], }; unsafe { ZJIT_STATE = Enabled(zjit_state); } @@ -186,6 +191,10 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } + pub fn get_jit_frames() -> &'static mut Vec<*mut JITFrame> { + &mut ZJITState::get_instance().jit_frames + } + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { &ZJITState::get_instance().method_annotations } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index b2e67afb166b9c..537813fee7438d 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -163,6 +163,7 @@ make_counters! { gc_time_ns, invalidation_time_ns, + compiled_side_exit_count, side_exit_size, compile_side_exit_time_ns, @@ -236,6 +237,7 @@ make_counters! { exit_splatkw_not_profiled, exit_directive_induced, exit_send_while_tracing, + exit_invokeblock_not_ifunc, } // Send fallback counters that are summed as dynamic_send_count @@ -285,6 +287,10 @@ make_counters! { send_fallback_super_target_not_found, send_fallback_super_target_complex_args_pass, send_fallback_cannot_send_direct, + send_fallback_invokeblock_not_specialized, + send_fallback_sendforward_not_specialized, + send_fallback_invokesuperforward_not_specialized, + send_fallback_single_ractor_mode_required, send_fallback_uncategorized, } @@ -629,6 +635,7 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { => exit_patchpoint_root_box_only, SendWhileTracing => exit_send_while_tracing, NoProfileSend => exit_no_profile_send, + InvokeBlockNotIfunc => exit_invokeblock_not_ifunc, } } @@ -682,6 +689,10 @@ pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter SuperPolymorphic => send_fallback_super_polymorphic, SuperTargetNotFound => send_fallback_super_target_not_found, SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + InvokeBlockNotSpecialized => send_fallback_invokeblock_not_specialized, + SendForwardNotSpecialized => send_fallback_sendforward_not_specialized, + InvokeSuperForwardNotSpecialized => send_fallback_invokesuperforward_not_specialized, + SingleRactorModeRequired => send_fallback_single_ractor_mode_required, Uncategorized(_) => send_fallback_uncategorized, } }