From c42a6338f4041a918773ccd75dd987be14085bfa Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 13:05:04 -0700 Subject: [PATCH 01/23] ZJIT: Check native stack in function_stub_hit (#16585) When JIT-to-JIT calls hit a function stub for lazy compilation, check if the native stack is nearly exhausted before proceeding. If so, bail out to the interpreter instead of compiling and entering more JIT code. This complements the check in rb_zjit_iseq_gen_entry_point (which guards interpreter-to-JIT entry) by also guarding JIT-to-JIT entry via function stubs, matching what YJIT does in branch_stub_hit. --- zjit/src/codegen.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 9c4a456af91d2f..987e55de968f95 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2971,7 +2971,7 @@ c_callable! { /// This function is expected to be called repeatedly when ZJIT fails to compile the stub. /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. - fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE) -> *const u8 { + fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE, ec: EcPtr) -> *const u8 { with_vm_lock(src_loc!(), || { // gen_push_frame() doesn't set PC, so we need to set them before exit. // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. @@ -3001,15 +3001,21 @@ c_callable! { } } - // If we already know we can't compile the ISEQ, fail early without cb.mark_all_executable(). + // If we already know we can't compile the ISEQ, or there is insufficient native + // stack space, fail early without cb.mark_all_executable(). // TODO: Alan thinks the payload status part of this check can happen without the VM lock, since the whole // code path can be made read-only. But you still need the check as is while holding the VM lock in any case. let cb = ZJITState::get_code_block(); + let native_stack_full = unsafe { rb_ec_stack_check(ec as _) } != 0; let payload = get_or_create_iseq_payload(iseq); let last_status = payload.versions.last().map(|version| &unsafe { version.as_ref() }.status); let compile_error = match last_status { Some(IseqStatus::CantCompile(err)) => Some(err), _ if cb.has_dropped_bytes() => Some(&CompileError::OutOfMemory), + _ if native_stack_full => { + incr_counter!(skipped_native_stack_full); + Some(&CompileError::NativeStackTooLarge) + }, _ => None, }; if let Some(compile_error) = compile_error { @@ -3114,7 +3120,7 @@ pub fn gen_function_stub_hit_trampoline(cb: &mut CodeBlock) -> Result Date: Fri, 27 Mar 2026 12:12:25 -0700 Subject: [PATCH 02/23] ZJIT: Categorize uncategorized send fallback reasons in stats Add specific SendFallbackReason variants for InvokeBlock, SendForward, and InvokeSuperForward instructions that previously stayed as Uncategorized in type_specialize. Also add SingleRactorModeRequired for paths in Send handling that failed due to multi-ractor mode without setting a reason. This reduces the "uncategorized" percentage from 53.6% to 0.2% on the lobsters benchmark, making the stats output actionable. --- zjit/src/hir.rs | 32 ++++++++++++++++++++++++++++++++ zjit/src/hir/opt_tests.rs | 4 ++-- zjit/src/stats.rs | 8 ++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 7107f7046afbb9..cd0075034438f2 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -698,6 +698,16 @@ pub enum SendFallbackReason { SuperPolymorphic, /// The `super` target call uses a complex argument pattern that the optimizer does not support. SuperTargetComplexArgsPass, + /// The `invokeblock` instruction is not yet optimized in `type_specialize`. + InvokeBlockNotSpecialized, + /// The `sendforward` instruction (argument forwarding `...`) is not yet optimized in + /// `type_specialize`. + SendForwardNotSpecialized, + /// The `invokesuperforward` instruction (super with forwarding `...`) is not yet optimized in + /// `type_specialize`. + InvokeSuperForwardNotSpecialized, + /// The single-ractor-mode assumption could not be made. + SingleRactorModeRequired, /// Initial fallback reason for every instruction, which should be mutated to /// a more actionable reason when an attempt to specialize the instruction fails. Uncategorized(ruby_vminsn_type), @@ -745,6 +755,10 @@ impl Display for SendFallbackReason { SuperPolymorphic => write!(f, "super: polymorphic call site"), SuperTargetNotFound => write!(f, "super: profiled target method cannot be found"), SuperTargetComplexArgsPass => write!(f, "super: complex argument passing to `super` target call"), + InvokeBlockNotSpecialized => write!(f, "InvokeBlock: not yet specialized"), + SendForwardNotSpecialized => write!(f, "SendForward: not yet specialized"), + InvokeSuperForwardNotSpecialized => write!(f, "InvokeSuperForward: not yet specialized"), + SingleRactorModeRequired => write!(f, "Single-ractor mode required"), Uncategorized(insn) => write!(f, "Uncategorized({})", insn_name(*insn as usize)), } } @@ -3716,6 +3730,7 @@ impl Function { // Check for "defined with an un-shareable Proc in a different Ractor" if !procv.shareable_p() && !self.assume_single_ractor_mode(block, state) { // TODO(alan): Turn this into a ractor belonging guard to work better in multi ractor mode. + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } // Check singleton class assumption first, before emitting other patchpoints @@ -3735,6 +3750,7 @@ impl Function { // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } // Check singleton class assumption first, before emitting other patchpoints @@ -3755,6 +3771,7 @@ impl Function { // Check if we're accessing ivars of a Class or Module object as they require single-ractor mode. // We omit gen_prepare_non_leaf_call on gen_getivar, so it's unsafe to raise for multi-ractor mode. if klass.is_metaclass() && !self.assume_single_ractor_mode(block, state) { + self.set_dynamic_send_reason(insn_id, SingleRactorModeRequired); self.push_insn_id(block, insn_id); continue; } @@ -3802,12 +3819,15 @@ impl Function { { let native_index = (index as i64) * (SIZEOF_VALUE as i64); if native_index > (i32::MAX as i64) { + self.set_dynamic_send_reason(insn_id, TooManyArgsForLir); self.push_insn_id(block, insn_id); continue; } } // Get the profiled type to check if the fields is embedded or heap allocated. let Some(is_embedded) = self.profiled_type_of_at(recv, frame_state.insn_idx).map(|t| t.flags().is_struct_embedded()) else { // No (monomorphic/skewed polymorphic) profile info + let reason = if has_block { SendNoProfiles } else { SendWithoutBlockNoProfiles }; + self.set_dynamic_send_reason(insn_id, reason); self.push_insn_id(block, insn_id); continue; }; // Check singleton class assumption first, before emitting other patchpoints @@ -4230,6 +4250,18 @@ impl Function { continue; } } + Insn::InvokeBlock { .. } => { + self.set_dynamic_send_reason(insn_id, InvokeBlockNotSpecialized); + self.push_insn_id(block, insn_id); + } + Insn::SendForward { .. } => { + self.set_dynamic_send_reason(insn_id, SendForwardNotSpecialized); + self.push_insn_id(block, insn_id); + } + Insn::InvokeSuperForward { .. } => { + self.set_dynamic_send_reason(insn_id, InvokeSuperForwardNotSpecialized); + self.push_insn_id(block, insn_id); + } _ => { self.push_insn_id(block, insn_id); } } } diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 3cf02934e7ad23..40d126d56f1638 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -7517,7 +7517,7 @@ mod hir_opt_tests { v4:BasicObject = LoadArg :self@0 Jump bb3(v4) bb3(v6:BasicObject): - v11:BasicObject = Send v6, :foo # SendFallbackReason: Uncategorized(opt_send_without_block) + v11:BasicObject = Send v6, :foo # SendFallbackReason: Single-ractor mode required CheckInterrupts Return v11 "); @@ -14810,7 +14810,7 @@ mod hir_opt_tests { v88:Array = RefineType v67, Array v89:CInt64 = UnboxFixnum v68 v90:BasicObject = ArrayAref v88, v89 - v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: Uncategorized(invokeblock) + v74:BasicObject = InvokeBlock, v90 # SendFallbackReason: InvokeBlock: not yet specialized v91:Fixnum[1] = Const Value(1) v92:Fixnum = FixnumAdd v68, v91 PatchPoint NoEPEscape(each) diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index b2e67afb166b9c..dae1d959b7dc86 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -285,6 +285,10 @@ make_counters! { send_fallback_super_target_not_found, send_fallback_super_target_complex_args_pass, send_fallback_cannot_send_direct, + send_fallback_invokeblock_not_specialized, + send_fallback_sendforward_not_specialized, + send_fallback_invokesuperforward_not_specialized, + send_fallback_single_ractor_mode_required, send_fallback_uncategorized, } @@ -682,6 +686,10 @@ pub fn send_fallback_counter(reason: crate::hir::SendFallbackReason) -> Counter SuperPolymorphic => send_fallback_super_polymorphic, SuperTargetNotFound => send_fallback_super_target_not_found, SuperTargetComplexArgsPass => send_fallback_super_target_complex_args_pass, + InvokeBlockNotSpecialized => send_fallback_invokeblock_not_specialized, + SendForwardNotSpecialized => send_fallback_sendforward_not_specialized, + InvokeSuperForwardNotSpecialized => send_fallback_invokesuperforward_not_specialized, + SingleRactorModeRequired => send_fallback_single_ractor_mode_required, Uncategorized(_) => send_fallback_uncategorized, } } From 6322acbd025c30004797163da1949819b9b281b1 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 12:45:00 -0700 Subject: [PATCH 03/23] ZJIT: Set fallback reasons at HIR construction instead of type_specialize Move the InvokeBlockNotSpecialized, SendForwardNotSpecialized, and InvokeSuperForwardNotSpecialized reasons from type_specialize match arms to HIR construction, since type_specialize doesn't attempt to specialize these instructions anyway. --- zjit/src/hir.rs | 18 +++--------------- zjit/src/hir/tests.rs | 18 +++++++++--------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index cd0075034438f2..824d89b5657229 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -4250,18 +4250,6 @@ impl Function { continue; } } - Insn::InvokeBlock { .. } => { - self.set_dynamic_send_reason(insn_id, InvokeBlockNotSpecialized); - self.push_insn_id(block, insn_id); - } - Insn::SendForward { .. } => { - self.set_dynamic_send_reason(insn_id, SendForwardNotSpecialized); - self.push_insn_id(block, insn_id); - } - Insn::InvokeSuperForward { .. } => { - self.set_dynamic_send_reason(insn_id, InvokeSuperForwardNotSpecialized); - self.push_insn_id(block, insn_id); - } _ => { self.push_insn_id(block, insn_id); } } } @@ -8016,7 +8004,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; let recv = state.stack_pop()?; - let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + let send_forward = fun.push_insn(block, Insn::SendForward { recv, cd, blockiseq, args, state: exit_id, reason: SendForwardNotSpecialized }); state.stack_push(send_forward); if !blockiseq.is_null() { @@ -8088,7 +8076,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let argc = unsafe { vm_ci_argc((*cd).ci) }; let args = state.stack_pop_n(argc as usize + usize::from(forwarding))?; let recv = state.stack_pop()?; - let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: Uncategorized(opcode) }); + let result = fun.push_insn(block, Insn::InvokeSuperForward { recv, cd, blockiseq, args, state: exit_id, reason: InvokeSuperForwardNotSpecialized }); state.stack_push(result); if !blockiseq.is_null() { @@ -8123,7 +8111,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let argc = unsafe { vm_ci_argc((*cd).ci) }; let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; - let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: Uncategorized(opcode) }); + let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }); state.stack_push(result); } YARVINSN_getglobal => { diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 606df902f20183..72012b7eba9504 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -2240,7 +2240,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized CheckInterrupts Return v16 "); @@ -2265,7 +2265,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized v17:CPtr = GetEP 0 v18:BasicObject = LoadField v17, :...@0x1010 CheckInterrupts @@ -2292,7 +2292,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v16:BasicObject = InvokeSuperForward v9, 0x1008, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized v18:Fixnum[1] = Const Value(1) v21:BasicObject = Send v16, :+, v18 # SendFallbackReason: Uncategorized(opt_plus) CheckInterrupts @@ -2320,7 +2320,7 @@ pub mod hir_build_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) - v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: Uncategorized(invokesuperforward) + v18:BasicObject = InvokeSuperForward v9, 0x1008, v15, v10 # SendFallbackReason: InvokeSuperForward: not yet specialized CheckInterrupts Return v18 "); @@ -2431,7 +2431,7 @@ pub mod hir_build_tests { v7:BasicObject = LoadArg :...@1 Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): - v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: Uncategorized(sendforward) + v16:BasicObject = SendForward v9, 0x1008, :foo, v10 # SendFallbackReason: SendForward: not yet specialized CheckInterrupts Return v16 "); @@ -4369,7 +4369,7 @@ pub mod hir_build_tests { v42 = RefineType v38, Falsy IfFalse v41, bb4(v18, v19, v20, v21, v22, v27) v44:ObjectSubclass[BlockParamProxy] = RefineType v38, Truthy - v48:BasicObject = InvokeBlock, v27 # SendFallbackReason: Uncategorized(invokeblock) + v48:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized v51:BasicObject = InvokeBuiltin dir_s_close, v18, v27 CheckInterrupts Return v48 @@ -4723,7 +4723,7 @@ pub mod hir_build_tests { v4:BasicObject = LoadArg :self@0 Jump bb3(v4) bb3(v6:BasicObject): - v10:BasicObject = InvokeBlock # SendFallbackReason: Uncategorized(invokeblock) + v10:BasicObject = InvokeBlock # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts Return v10 "); @@ -4752,7 +4752,7 @@ pub mod hir_build_tests { v9:BasicObject = LoadArg :y@2 Jump bb3(v7, v8, v9) bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): - v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: Uncategorized(invokeblock) + v19:BasicObject = InvokeBlock, v12, v13 # SendFallbackReason: InvokeBlock: not yet specialized CheckInterrupts Return v19 "); @@ -5038,7 +5038,7 @@ pub mod hir_build_tests { Return v48 bb7(v67:BasicObject, v68:Fixnum): v72:BasicObject = InvokeBuiltin rb_jit_ary_at, v67, v68 - v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: Uncategorized(invokeblock) + v74:BasicObject = InvokeBlock, v72 # SendFallbackReason: InvokeBlock: not yet specialized v78:Fixnum = InvokeBuiltin rb_jit_fixnum_inc, v67, v68 PatchPoint NoEPEscape(each) Jump bb8(v67, v78) From 9a3fedd3dcd39a39c1162f5e533835bda126a44d Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Fri, 27 Mar 2026 21:34:38 +0100 Subject: [PATCH 04/23] [ruby/prism] Nodoc the ripper shim It appears in documentation on https://docs.ruby-lang.org/ under `Object::Ripper` https://github.com/ruby/prism/commit/6c1000661c --- lib/prism/translation/ripper/shim.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/prism/translation/ripper/shim.rb b/lib/prism/translation/ripper/shim.rb index 10e21cd16a213f..038235e078f2d8 100644 --- a/lib/prism/translation/ripper/shim.rb +++ b/lib/prism/translation/ripper/shim.rb @@ -2,4 +2,4 @@ # This writes the prism ripper translation into the Ripper constant so that # users can transparently use Ripper without any changes. -Ripper = Prism::Translation::Ripper +Ripper = Prism::Translation::Ripper # :nodoc: From 6519a7749631538a74b63e182be9a171c1afdc84 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Sun, 22 Mar 2026 12:56:29 -0700 Subject: [PATCH 05/23] ZJIT: Specialize invokeblock for IFUNC block handlers When profiling shows a monomorphic IFUNC block handler, emit InvokeBlockIfunc which calls rb_vm_yield_with_cfunc directly instead of falling back to rb_vm_invokeblock. This avoids the interpreter dispatch overhead for the most common invokeblock case on lobsters (77% of all invokeblock calls). The implementation: - Loads the block handler from the local EP - Guards tag bits (& 0x3 == 0x3) to confirm IFUNC - Untags the pointer to get the captured block - Calls rb_vm_yield_with_cfunc(ec, captured, argc, argv) --- zjit/src/codegen.rs | 37 +++++++++++++++++++++ zjit/src/hir.rs | 78 ++++++++++++++++++++++++++++++++++++++++++++- zjit/src/stats.rs | 2 ++ 3 files changed, 116 insertions(+), 1 deletion(-) diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 987e55de968f95..b4b66e52407d53 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -623,6 +623,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::InvokeSuper { cd, blockiseq, state, reason, .. } => gen_invokesuper(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeSuperForward { cd, blockiseq, state, reason, .. } => gen_invokesuperforward(jit, asm, cd, blockiseq, &function.frame_state(state), reason), &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), + Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), // Ensure we have enough room fit ec, self, and arguments // TODO remove this check when we have stack args (we can use Time.new to test it) @@ -1703,6 +1704,42 @@ fn gen_invokeblock( ) } +/// Compile invokeblock for IFUNC block handlers. +/// Calls rb_vm_yield_with_cfunc directly. +fn gen_invokeblock_ifunc( + jit: &mut JITState, + asm: &mut Assembler, + cd: *const rb_call_data, + block_handler: Opnd, + args: Vec, + state: &FrameState, +) -> lir::Opnd { + let _ = cd; // cd is not needed for the direct call + + gen_prepare_non_leaf_call(jit, asm, state); + + // Push args to memory so we can pass argv pointer + let argv_ptr = gen_push_opnds(asm, &args); + + // Untag the block handler to get the captured block pointer + // captured = block_handler & ~0x3 + asm_comment!(asm, "untag block handler to get captured block"); + let captured = asm.and(block_handler, Opnd::Imm(!0x3i64)); + + asm_comment!(asm, "call rb_vm_yield_with_cfunc"); + unsafe extern "C" { + fn rb_vm_yield_with_cfunc( + ec: EcPtr, + captured: VALUE, + argc: i32, + argv: *const VALUE, + ) -> VALUE; + } + let result = asm_ccall!(asm, rb_vm_yield_with_cfunc, EC, captured, (args.len() as i64).into(), argv_ptr); + gen_pop_opnds(asm, &args); + result +} + fn gen_invokeproc( jit: &mut JITState, asm: &mut Assembler, diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 824d89b5657229..90d6682af8fd52 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -538,6 +538,7 @@ pub enum SideExitReason { DirectiveInduced, SendWhileTracing, NoProfileSend, + InvokeBlockNotIfunc, } #[derive(Debug, Clone, Copy)] @@ -1011,6 +1012,14 @@ pub enum Insn { state: InsnId, reason: SendFallbackReason, }, + /// Optimized invokeblock for IFUNC block handlers. + /// Calls rb_vm_yield_with_cfunc directly instead of going through rb_vm_invokeblock. + InvokeBlockIfunc { + cd: *const rb_call_data, + block_handler: InsnId, + args: Vec, + state: InsnId, + }, /// Call Proc#call optimized method type. InvokeProc { recv: InsnId, @@ -1335,6 +1344,11 @@ macro_rules! for_each_operand_impl { $visit_many!(args); $visit_one!(state); } + Insn::InvokeBlockIfunc { block_handler, args, state, .. } => { + $visit_one!(block_handler); + $visit_many!(args); + $visit_one!(state); + } Insn::CCall { recv, args, .. } => { $visit_one!(recv); $visit_many!(args); @@ -1583,6 +1597,7 @@ impl Insn { Insn::InvokeSuper { .. } => effects::Any, Insn::InvokeSuperForward { .. } => effects::Any, Insn::InvokeBlock { .. } => effects::Any, + Insn::InvokeBlockIfunc { .. } => effects::Any, Insn::SendDirect { .. } => effects::Any, Insn::InvokeBuiltin { .. } => effects::Any, Insn::EntryPoint { .. } => effects::Any, @@ -1946,6 +1961,13 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { write!(f, " # SendFallbackReason: {reason}")?; Ok(()) } + Insn::InvokeBlockIfunc { block_handler, args, .. } => { + write!(f, "InvokeBlockIfunc {block_handler}")?; + for arg in args { + write!(f, ", {arg}")?; + } + Ok(()) + } Insn::InvokeProc { recv, args, kw_splat, .. } => { write!(f, "InvokeProc {recv}")?; for arg in args { @@ -2834,6 +2856,12 @@ impl Function { state, reason, }, + &InvokeBlockIfunc { cd, block_handler, ref args, state } => InvokeBlockIfunc { + cd, + block_handler: find!(block_handler), + args: find_vec!(args), + state: find!(state), + }, &InvokeProc { recv, ref args, state, kw_splat } => InvokeProc { recv: find!(recv), args: find_vec!(args), @@ -3039,6 +3067,7 @@ impl Function { Insn::InvokeSuper { .. } => types::BasicObject, Insn::InvokeSuperForward { .. } => types::BasicObject, Insn::InvokeBlock { .. } => types::BasicObject, + Insn::InvokeBlockIfunc { .. } => types::BasicObject, Insn::InvokeProc { .. } => types::BasicObject, Insn::InvokeBuiltin { return_type, .. } => return_type.unwrap_or(types::BasicObject), Insn::Defined { pushval, .. } => Type::from_value(*pushval).union(types::NilClass), @@ -6129,6 +6158,7 @@ impl Function { } // Instructions with a Vec of Ruby objects Insn::InvokeBlock { ref args, .. } + | Insn::InvokeBlockIfunc { ref args, .. } | Insn::NewArray { elements: ref args, .. } | Insn::ArrayHash { elements: ref args, .. } | Insn::ArrayMin { elements: ref args, .. } @@ -8111,7 +8141,53 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { let argc = unsafe { vm_ci_argc((*cd).ci) }; let block_arg = (flags & VM_CALL_ARGS_BLOCKARG) != 0; let args = state.stack_pop_n(argc as usize + usize::from(block_arg))?; - let result = fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }); + + // Check if this is a monomorphic IFUNC block handler we can specialize + let block_handler_types = profiles.payload.profile.get_operand_types(exit_state.insn_idx); + let is_ifunc = (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT)) == 0 + && block_handler_types.is_some_and(|types| types.len() == 1 && { + let summary = TypeDistributionSummary::new(&types[0]); + summary.is_monomorphic() && unsafe { rb_IMEMO_TYPE_P(summary.bucket(0).class(), imemo_ifunc) == 1 } + }); + + let result = if is_ifunc { + // Get the local EP to load the block handler + let level = get_lvar_level(fun.iseq); + let lep = fun.push_insn(block, Insn::GetEP { level }); + let block_handler = fun.push_insn(block, Insn::LoadField { + recv: lep, + id: ID!(_env_data_index_specval), + offset: SIZEOF_VALUE_I32 * VM_ENV_DATA_INDEX_SPECVAL, + return_type: types::CInt64, + }); + + // Guard that the block handler is an IFUNC (tag bits & 0x3 == 0x3) + // bit 0x1 must be set (iseq or ifunc) + fun.push_insn(block, Insn::GuardAnyBitSet { + val: block_handler, + mask: Const::CUInt64(0x1), + mask_name: None, + reason: SideExitReason::InvokeBlockNotIfunc, + state: exit_id, + }); + // bit 0x2 must be set (ifunc specifically, not iseq) + fun.push_insn(block, Insn::GuardAnyBitSet { + val: block_handler, + mask: Const::CUInt64(0x2), + mask_name: None, + reason: SideExitReason::InvokeBlockNotIfunc, + state: exit_id, + }); + + fun.push_insn(block, Insn::InvokeBlockIfunc { + cd, + block_handler, + args, + state: exit_id, + }) + } else { + fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }) + }; state.stack_push(result); } YARVINSN_getglobal => { diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index dae1d959b7dc86..330b9f32bfbe2f 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -236,6 +236,7 @@ make_counters! { exit_splatkw_not_profiled, exit_directive_induced, exit_send_while_tracing, + exit_invokeblock_not_ifunc, } // Send fallback counters that are summed as dynamic_send_count @@ -633,6 +634,7 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { => exit_patchpoint_root_box_only, SendWhileTracing => exit_send_while_tracing, NoProfileSend => exit_no_profile_send, + InvokeBlockNotIfunc => exit_invokeblock_not_ifunc, } } From 16e96105a75d455e9f6666ff63bf854e6ccb619d Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 13:23:03 -0700 Subject: [PATCH 06/23] ZJIT: Add tests for invokeblock IFUNC specialization Test Enumerable#map on a class with a Ruby `each` that uses `yield`. This exercises the IFUNC block handler path in invokeblock, where Enumerable C code creates an IFUNC wrapping the user's block. --- zjit/src/codegen_tests.rs | 17 +++++++++++++++++ zjit/src/hir/opt_tests.rs | 40 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 3a21b001cd13c6..db94a32036ad4e 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -4700,6 +4700,23 @@ fn test_invokeblock_multiple_yields() { "), @"[1, 2, 3]"); } +#[test] +fn test_invokeblock_ifunc_map() { + eval(" + class MyList + include Enumerable + def each + yield 1 + yield 2 + yield 3 + end + end + def test = MyList.new.map { |x| x * 2 } + test + "); + assert_snapshot!(assert_compiles("test"), @"[2, 4, 6]"); +} + #[test] fn test_ccall_variadic_with_multiple_args() { eval(" diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 40d126d56f1638..315467ca72d0c8 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -15075,4 +15075,44 @@ mod hir_opt_tests { Return v36 "); } + + #[test] + fn test_invokeblock_ifunc() { + eval(" + class IFuncTestList + include Enumerable + def each + yield 1 + yield 2 + end + end + IFuncTestList.new.map { |x| x } + "); + assert_snapshot!(hir_string_proc("IFuncTestList.instance_method(:each)"), @" + fn each@:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + Jump bb3(v1) + bb2(): + EntryPoint JIT(0) + v4:BasicObject = LoadArg :self@0 + Jump bb3(v4) + bb3(v6:BasicObject): + v10:Fixnum[1] = Const Value(1) + v12:CPtr = GetEP 0 + v13:CInt64 = LoadField v12, :_env_data_index_specval@0x1000 + v14:CInt64 = GuardAnyBitSet v13, CUInt64(1) + v15:CInt64 = GuardAnyBitSet v13, CUInt64(2) + v16:BasicObject = InvokeBlockIfunc v13, v10 + v20:Fixnum[2] = Const Value(2) + v22:CPtr = GetEP 0 + v23:CInt64 = LoadField v22, :_env_data_index_specval@0x1000 + v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) + v25:CInt64 = GuardAnyBitSet v23, CUInt64(2) + v26:BasicObject = InvokeBlockIfunc v23, v20 + CheckInterrupts + Return v26 + "); + } } From cee3d5f45bb74fdb3d93c48f22a7a622bc435cb3 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 13:58:34 -0700 Subject: [PATCH 07/23] ZJIT: Combine two IFUNC tag guards into one Use IntAnd + GuardBitEquals to check (block_handler & 0x3) == 0x3, matching VM_BH_IFUNC_P() in the interpreter. Add the IntAnd HIR instruction (placed next to IntOr) for bitwise AND on raw C integers. --- zjit/src/codegen.rs | 1 + zjit/src/hir.rs | 30 +++++++++++++++--------------- zjit/src/hir/opt_tests.rs | 22 ++++++++++++---------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index b4b66e52407d53..b21d4070d2a222 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -644,6 +644,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio Insn::FixnumAnd { left, right } => gen_fixnum_and(asm, opnd!(left), opnd!(right)), Insn::FixnumOr { left, right } => gen_fixnum_or(asm, opnd!(left), opnd!(right)), Insn::FixnumXor { left, right } => gen_fixnum_xor(asm, opnd!(left), opnd!(right)), + Insn::IntAnd { left, right } => asm.and(opnd!(left), opnd!(right)), Insn::IntOr { left, right } => gen_int_or(asm, opnd!(left), opnd!(right)), &Insn::FixnumLShift { left, right, state } => { // We only create FixnumLShift when we know the shift amount statically and it's in [0, diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 90d6682af8fd52..097e1b824be664 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -1072,6 +1072,7 @@ pub enum Insn { FixnumAnd { left: InsnId, right: InsnId }, FixnumOr { left: InsnId, right: InsnId }, FixnumXor { left: InsnId, right: InsnId }, + IntAnd { left: InsnId, right: InsnId }, IntOr { left: InsnId, right: InsnId }, FixnumLShift { left: InsnId, right: InsnId, state: InsnId }, FixnumRShift { left: InsnId, right: InsnId }, @@ -1275,6 +1276,7 @@ macro_rules! for_each_operand_impl { | Insn::FixnumAnd { left, right } | Insn::FixnumOr { left, right } | Insn::FixnumXor { left, right } + | Insn::IntAnd { left, right } | Insn::IntOr { left, right } | Insn::FixnumRShift { left, right } | Insn::IsBitEqual { left, right } @@ -1617,6 +1619,7 @@ impl Insn { Insn::FixnumAnd { .. } => effects::Empty, Insn::FixnumOr { .. } => effects::Empty, Insn::FixnumXor { .. } => effects::Empty, + Insn::IntAnd { .. } => effects::Empty, Insn::IntOr { .. } => effects::Empty, Insn::FixnumLShift { .. } => effects::Empty, Insn::FixnumRShift { .. } => effects::Empty, @@ -2006,6 +2009,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Insn::FixnumAnd { left, right, .. } => { write!(f, "FixnumAnd {left}, {right}") }, Insn::FixnumOr { left, right, .. } => { write!(f, "FixnumOr {left}, {right}") }, Insn::FixnumXor { left, right, .. } => { write!(f, "FixnumXor {left}, {right}") }, + Insn::IntAnd { left, right } => { write!(f, "IntAnd {left}, {right}") }, Insn::IntOr { left, right } => { write!(f, "IntOr {left}, {right}") }, Insn::FixnumLShift { left, right, .. } => { write!(f, "FixnumLShift {left}, {right}") }, Insn::FixnumRShift { left, right, .. } => { write!(f, "FixnumRShift {left}, {right}") }, @@ -2795,6 +2799,7 @@ impl Function { &FixnumAnd { left, right } => FixnumAnd { left: find!(left), right: find!(right) }, &FixnumOr { left, right } => FixnumOr { left: find!(left), right: find!(right) }, &FixnumXor { left, right } => FixnumXor { left: find!(left), right: find!(right) }, + &IntAnd { left, right } => IntAnd { left: find!(left), right: find!(right) }, &IntOr { left, right } => IntOr { left: find!(left), right: find!(right) }, &FixnumLShift { left, right, state } => FixnumLShift { left: find!(left), right: find!(right), state }, &FixnumRShift { left, right } => FixnumRShift { left: find!(left), right: find!(right) }, @@ -3057,6 +3062,7 @@ impl Function { Insn::FixnumAnd { .. } => types::Fixnum, Insn::FixnumOr { .. } => types::Fixnum, Insn::FixnumXor { .. } => types::Fixnum, + Insn::IntAnd { .. } => types::CInt64, Insn::IntOr { left, .. } => self.type_of(*left).unspecialized(), Insn::FixnumLShift { .. } => types::Fixnum, Insn::FixnumRShift { .. } => types::Fixnum, @@ -6248,7 +6254,8 @@ impl Function { Err(ValidationError::MiscValidationError(insn_id, "IsBitEqual can only compare CInt/CInt or RubyValue/RubyValue".to_string())) } } - Insn::IntOr { left, right } => { + Insn::IntAnd { left, right } + | Insn::IntOr { left, right } => { // TODO: Expand this to other matching C integer sizes when we need them. self.assert_subtype(insn_id, left, types::CInt64)?; self.assert_subtype(insn_id, right, types::CInt64) @@ -8161,20 +8168,13 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { return_type: types::CInt64, }); - // Guard that the block handler is an IFUNC (tag bits & 0x3 == 0x3) - // bit 0x1 must be set (iseq or ifunc) - fun.push_insn(block, Insn::GuardAnyBitSet { - val: block_handler, - mask: Const::CUInt64(0x1), - mask_name: None, - reason: SideExitReason::InvokeBlockNotIfunc, - state: exit_id, - }); - // bit 0x2 must be set (ifunc specifically, not iseq) - fun.push_insn(block, Insn::GuardAnyBitSet { - val: block_handler, - mask: Const::CUInt64(0x2), - mask_name: None, + // Guard that the block handler is an IFUNC (tag bits & 0x3 == 0x3), + // matching VM_BH_IFUNC_P() in the interpreter. + let tag_mask = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let tag_bits = fun.push_insn(block, Insn::IntAnd { left: block_handler, right: tag_mask }); + fun.push_insn(block, Insn::GuardBitEquals { + val: tag_bits, + expected: Const::CInt64(0x3), reason: SideExitReason::InvokeBlockNotIfunc, state: exit_id, }); diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 315467ca72d0c8..c9f2a45b048adc 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -15102,17 +15102,19 @@ mod hir_opt_tests { v10:Fixnum[1] = Const Value(1) v12:CPtr = GetEP 0 v13:CInt64 = LoadField v12, :_env_data_index_specval@0x1000 - v14:CInt64 = GuardAnyBitSet v13, CUInt64(1) - v15:CInt64 = GuardAnyBitSet v13, CUInt64(2) - v16:BasicObject = InvokeBlockIfunc v13, v10 - v20:Fixnum[2] = Const Value(2) - v22:CPtr = GetEP 0 - v23:CInt64 = LoadField v22, :_env_data_index_specval@0x1000 - v24:CInt64 = GuardAnyBitSet v23, CUInt64(1) - v25:CInt64 = GuardAnyBitSet v23, CUInt64(2) - v26:BasicObject = InvokeBlockIfunc v23, v20 + v14:CInt64[3] = Const CInt64(3) + v15:CInt64 = IntAnd v13, v14 + v16:CInt64[3] = GuardBitEquals v15, CInt64(3) + v17:BasicObject = InvokeBlockIfunc v13, v10 + v21:Fixnum[2] = Const Value(2) + v23:CPtr = GetEP 0 + v24:CInt64 = LoadField v23, :_env_data_index_specval@0x1000 + v25:CInt64[3] = Const CInt64(3) + v26:CInt64 = IntAnd v24, v25 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) + v28:BasicObject = InvokeBlockIfunc v24, v21 CheckInterrupts - Return v26 + Return v28 "); } } From 2d2351af56569fc3cd9213c35aaabd30e82e39fe Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sat, 28 Mar 2026 00:25:00 +0100 Subject: [PATCH 08/23] Remove unused compile options/jobs (#16583) SYMBOL_GC removed in https://github.com/ruby/ruby/commit/2bcb155b49bb421ee82c0d5980546a5071113407 THREAD_CACHE removed in https://github.com/ruby/ruby/commit/be1bbd5b7d40ad863ab35097765d3754726bbd54 BIGNUM_DEBUG removed in https://github.com/ruby/ruby/commit/e60cd14d85b35c9e60485e640c08eebf539c1cfc --- .github/workflows/compilers.yml | 3 --- bignum.c | 5 ----- symbol.c | 7 ------- 3 files changed, 15 deletions(-) diff --git a/.github/workflows/compilers.yml b/.github/workflows/compilers.yml index f678eabd1c2ba4..0c3fce5bccb173 100644 --- a/.github/workflows/compilers.yml +++ b/.github/workflows/compilers.yml @@ -221,7 +221,6 @@ jobs: - { uses: './.github/actions/compilers', name: 'NDEBUG', with: { cppflags: '-DNDEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'RUBY_DEBUG', with: { cppflags: '-DRUBY_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'ARRAY_DEBUG', with: { cppflags: '-DARRAY_DEBUG' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'BIGNUM_DEBUG', with: { cppflags: '-DBIGNUM_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'CCAN_LIST_DEBUG', with: { cppflags: '-DCCAN_LIST_DEBUG' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'CPDEBUG=-1', with: { cppflags: '-DCPDEBUG=-1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'ENC_DEBUG', with: { cppflags: '-DENC_DEBUG' }, timeout-minutes: 5 } @@ -259,8 +258,6 @@ jobs: with: { sparse-checkout-cone-mode: false, sparse-checkout: /.github, persist-credentials: false } - { uses: './.github/actions/setup/directories', with: { srcdir: 'src', builddir: 'build', makeup: true, fetch-depth: 10 } } - { uses: './.github/actions/compilers', name: 'USE_LAZY_LOAD', with: { cppflags: '-DUSE_LAZY_LOAD' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'USE_SYMBOL_GC=0', with: { cppflags: '-DUSE_SYMBOL_GC=0' }, timeout-minutes: 5 } - - { uses: './.github/actions/compilers', name: 'USE_THREAD_CACHE=0', with: { cppflags: '-DUSE_THREAD_CACHE=0' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'USE_RUBY_DEBUG_LOG=1', with: { cppflags: '-DUSE_RUBY_DEBUG_LOG=1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'USE_DEBUG_COUNTER', with: { cppflags: '-DUSE_DEBUG_COUNTER=1' }, timeout-minutes: 5 } - { uses: './.github/actions/compilers', name: 'SHARABLE_MIDDLE_SUBSTRING', with: { cppflags: '-DSHARABLE_MIDDLE_SUBSTRING=1' }, timeout-minutes: 5 } diff --git a/bignum.c b/bignum.c index f5c58ab74312a4..11dff4d13927cf 100644 --- a/bignum.c +++ b/bignum.c @@ -2943,11 +2943,6 @@ bary_divmod(BDIGIT *qds, size_t qn, BDIGIT *rds, size_t rn, const BDIGIT *xds, s } } - -#ifndef BIGNUM_DEBUG -# define BIGNUM_DEBUG (0+RUBY_DEBUG) -#endif - static int bigzero_p(VALUE x) { diff --git a/symbol.c b/symbol.c index 85340be3a74c27..d3d7e13ea43626 100644 --- a/symbol.c +++ b/symbol.c @@ -27,13 +27,6 @@ #include "builtin.h" #include "ruby/internal/attr/nonstring.h" -#if defined(USE_SYMBOL_GC) && !(USE_SYMBOL_GC+0) -# undef USE_SYMBOL_GC -# define USE_SYMBOL_GC 0 -#else -# undef USE_SYMBOL_GC -# define USE_SYMBOL_GC 1 -#endif #if defined(SYMBOL_DEBUG) && (SYMBOL_DEBUG+0) # undef SYMBOL_DEBUG # define SYMBOL_DEBUG 1 From 1b2dfdf4beb531e6d1939e0d82975fab5ee580ff Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 16:25:04 -0700 Subject: [PATCH 09/23] Fix crash report being truncated by secondary crashes (#16586) When rb_bug() is called (e.g. from a ZJIT Rust panic), the crash report file could be truncated and left nearly empty due to two issues: 1. The crash report file was opened with default (buffered) I/O. If rb_vm_bugreport() triggered a secondary crash, buffered output was lost. Fix by setting the file to unbuffered with setvbuf(). 2. After rb_bug() wrote the crash report, die() called abort() which triggered our custom SIGABRT handler. The handler re-opened the same crash report file with "w" mode, truncating the report that was already written. Fix by resetting SIGABRT to SIG_DFL before abort(). --- error.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/error.c b/error.c index 140049fdd2cc2b..618e53fdc77ac2 100644 --- a/error.c +++ b/error.c @@ -908,6 +908,10 @@ bug_report_file(const char *file, int line, rb_pid_t *pid) int len = err_position_0(buf, sizeof(buf), file, line); if (out) { + /* Disable buffering so crash report output is not lost if + * rb_vm_bugreport() triggers a secondary crash (e.g. SIGSEGV + * while walking JIT frames). */ + setvbuf(out, NULL, _IONBF, 0); if ((ssize_t)fwrite(buf, 1, len, out) == (ssize_t)len) return out; fclose(out); } @@ -1081,6 +1085,10 @@ die(void) _set_abort_behavior( 0, _CALL_REPORTFAULT); #endif + /* Reset SIGABRT to default so that abort() does not trigger our custom + * handler (sigabrt), which would re-open the crash report file with "w" + * and truncate the report already written by rb_bug(). */ + signal(SIGABRT, SIG_DFL); abort(); } From c8f97943efcd34fc1a4fdc789289a916104025f9 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Feb 2026 11:08:14 -0800 Subject: [PATCH 10/23] ZJIT: Lightweight Frames Co-authored-by: Alan Wu --- cont.c | 10 +- depend | 7 + error.c | 3 +- eval.c | 5 +- gc.c | 10 +- jit.c | 4 + thread.c | 5 +- vm.c | 113 +++++++++---- vm_backtrace.c | 53 +++--- vm_core.h | 5 +- vm_dump.c | 31 ++-- vm_eval.c | 9 +- vm_exec.h | 4 + vm_insnhelper.c | 39 ++--- yjit.c | 2 +- zjit.h | 68 ++++++++ zjit.rb | 1 + zjit/bindgen/src/main.rs | 4 +- zjit/src/backend/arm64/mod.rs | 2 +- zjit/src/backend/lir.rs | 21 ++- zjit/src/backend/x86_64/mod.rs | 2 +- zjit/src/codegen.rs | 104 +++++++++--- zjit/src/cruby.rs | 7 - zjit/src/cruby_bindings.inc.rs | 12 ++ zjit/src/gc.rs | 28 +++- zjit/src/hir.rs | 2 +- zjit/src/jit_frame.rs | 288 +++++++++++++++++++++++++++++++++ zjit/src/lib.rs | 1 + zjit/src/payload.rs | 2 + zjit/src/state.rs | 9 ++ zjit/src/stats.rs | 1 + 31 files changed, 713 insertions(+), 139 deletions(-) create mode 100644 zjit/src/jit_frame.rs diff --git a/cont.c b/cont.c index 36ee34fe670f8b..9370058f554798 100644 --- a/cont.c +++ b/cont.c @@ -41,6 +41,7 @@ extern int madvise(caddr_t, size_t, int); #include "vm_sync.h" #include "id_table.h" #include "ractor_core.h" +#include "zjit.h" enum { DEBUG = 0, @@ -1455,8 +1456,11 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->pc && cfp->iseq && imemo_type((VALUE)cfp->iseq) == imemo_iseq) { - callback(cfp->iseq, data); + if (rb_zjit_cfp_has_pc(cfp) && rb_zjit_cfp_has_iseq(cfp)) { + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { + callback(iseq, data); + } } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); } @@ -1476,7 +1480,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (cfp->jit_return && cfp->jit_return != leave_exception) { + if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { ((rb_control_frame_t *)cfp)->jit_return = leave_exit; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/depend b/depend index f22dbbd8189aa0..6b5d7ecd40f494 100644 --- a/depend +++ b/depend @@ -2620,6 +2620,7 @@ cont.$(OBJEXT): {$(VPATH)}vm_debug.h cont.$(OBJEXT): {$(VPATH)}vm_opts.h cont.$(OBJEXT): {$(VPATH)}vm_sync.h cont.$(OBJEXT): {$(VPATH)}yjit.h +cont.$(OBJEXT): {$(VPATH)}zjit.h debug.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h debug.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h debug.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5420,6 +5421,7 @@ error.$(OBJEXT): {$(VPATH)}vm_opts.h error.$(OBJEXT): {$(VPATH)}vm_sync.h error.$(OBJEXT): {$(VPATH)}warning.rbinc error.$(OBJEXT): {$(VPATH)}yjit.h +error.$(OBJEXT): {$(VPATH)}zjit.h eval.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h eval.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h eval.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -5689,6 +5691,7 @@ eval.$(OBJEXT): {$(VPATH)}vm_core.h eval.$(OBJEXT): {$(VPATH)}vm_debug.h eval.$(OBJEXT): {$(VPATH)}vm_opts.h eval.$(OBJEXT): {$(VPATH)}vm_sync.h +eval.$(OBJEXT): {$(VPATH)}zjit.h explicit_bzero.$(OBJEXT): {$(VPATH)}config.h explicit_bzero.$(OBJEXT): {$(VPATH)}explicit_bzero.c explicit_bzero.$(OBJEXT): {$(VPATH)}internal/attr/format.h @@ -8072,6 +8075,7 @@ jit.$(OBJEXT): {$(VPATH)}vm_core.h jit.$(OBJEXT): {$(VPATH)}vm_debug.h jit.$(OBJEXT): {$(VPATH)}vm_opts.h jit.$(OBJEXT): {$(VPATH)}vm_sync.h +jit.$(OBJEXT): {$(VPATH)}zjit.h load.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h load.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h load.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -18617,6 +18621,7 @@ thread.$(OBJEXT): {$(VPATH)}vm_core.h thread.$(OBJEXT): {$(VPATH)}vm_debug.h thread.$(OBJEXT): {$(VPATH)}vm_opts.h thread.$(OBJEXT): {$(VPATH)}vm_sync.h +thread.$(OBJEXT): {$(VPATH)}zjit.h time.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h time.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h time.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20212,6 +20217,7 @@ vm_backtrace.$(OBJEXT): {$(VPATH)}vm_core.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_debug.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_opts.h vm_backtrace.$(OBJEXT): {$(VPATH)}vm_sync.h +vm_backtrace.$(OBJEXT): {$(VPATH)}zjit.h vm_dump.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_dump.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_dump.$(OBJEXT): $(CCAN_DIR)/list/list.h @@ -20460,6 +20466,7 @@ vm_dump.$(OBJEXT): {$(VPATH)}vm_core.h vm_dump.$(OBJEXT): {$(VPATH)}vm_debug.h vm_dump.$(OBJEXT): {$(VPATH)}vm_dump.c vm_dump.$(OBJEXT): {$(VPATH)}vm_opts.h +vm_dump.$(OBJEXT): {$(VPATH)}zjit.h vm_sync.$(OBJEXT): $(CCAN_DIR)/check_type/check_type.h vm_sync.$(OBJEXT): $(CCAN_DIR)/container_of/container_of.h vm_sync.$(OBJEXT): $(CCAN_DIR)/list/list.h diff --git a/error.c b/error.c index 618e53fdc77ac2..7f277017dd3ed9 100644 --- a/error.c +++ b/error.c @@ -50,6 +50,7 @@ #include "ruby_assert.h" #include "vm_core.h" #include "yjit.h" +#include "zjit.h" #include "builtin.h" @@ -2375,7 +2376,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method) rb_ivar_set(exc, id_name, method); err_init_recv(exc, recv); if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_ivar_set(exc, id_iseq, rb_iseqw_new(cfp->iseq)); + rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_zjit_cfp_iseq(cfp))); } return exc; } diff --git a/eval.c b/eval.c index 7d5ae75e3e6ee9..a30a36e4745a94 100644 --- a/eval.c +++ b/eval.c @@ -37,6 +37,7 @@ #include "ruby/vm.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" NORETURN(static void rb_raise_jump(VALUE, VALUE)); void rb_ec_clear_current_thread_trace_func(const rb_execution_context_t *ec); @@ -2010,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec) while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { if (VM_FRAME_RUBYFRAME_P(cfp)) { - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE) { + if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } - else if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_ENSURE && + else if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE && !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) && !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; diff --git a/gc.c b/gc.c index eb949bfae7c492..59e4a3f1fe7959 100644 --- a/gc.c +++ b/gc.c @@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && cfp->pc) { - const VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; - RUBY_ASSERT(cfp->pc >= iseq_encoded, "PC not set when allocating, breaking tracing"); - RUBY_ASSERT(cfp->pc <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_pc(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_size; + RUBY_ASSERT(rb_zjit_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(rb_zjit_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } diff --git a/jit.c b/jit.c index 11ff015e362c3d..0ff573f9e116c1 100644 --- a/jit.c +++ b/jit.c @@ -19,6 +19,7 @@ #include "internal/class.h" #include "internal/imemo.h" #include "ruby/internal/core/rtypeddata.h" +#include "zjit.h" #ifndef _WIN32 #include @@ -522,6 +523,9 @@ void rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) { cfp->pc = pc; + if (rb_zjit_enabled_p) { + cfp->jit_return = 0; + } } void diff --git a/thread.c b/thread.c index b13b06d7e5d94e..8586b4811a8df5 100644 --- a/thread.c +++ b/thread.c @@ -99,6 +99,7 @@ #include "ractor_core.h" #include "vm_debug.h" #include "vm_sync.h" +#include "zjit.h" #include "ccan/list/list.h" @@ -5915,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) VALUE num; void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset); if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) { - rb_iseq_clear_event_flags(cfp->iseq, cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); + rb_iseq_clear_event_flags(cfp->iseq, rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); rb_ary_push(lines, LONG2FIX(line + 1)); return; } @@ -5940,7 +5941,7 @@ update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg) if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES); if (branches) { - long pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; + long pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(cfp->iseq), pc)), count; VALUE counters = RARRAY_AREF(branches, 1); VALUE num = RARRAY_AREF(counters, idx); diff --git a/vm.c b/vm.c index 3825189f38ebac..71144b98e9b469 100644 --- a/vm.c +++ b/vm.c @@ -559,6 +559,8 @@ zjit_compile(rb_execution_context_t *ec) # define zjit_compile(ec) ((rb_jit_func_t)0) #endif +static inline void zjit_materialize_frames(rb_control_frame_t *cfp); + // Execute JIT code compiled by yjit_compile() or zjit_compile() static inline VALUE jit_exec(rb_execution_context_t *ec) @@ -578,7 +580,15 @@ jit_exec(rb_execution_context_t *ec) if (zjit_entry) { rb_jit_func_t func = zjit_compile(ec); if (func) { - return ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func); + // Materialize any remaining lightweight ZJIT frames on side exit. + // This is done here (once per JIT entry) instead of in each side exit + // to reduce generated code size. + if (UNDEF_P(result)) { + ec->cfp->jit_return = 0; + zjit_materialize_frames(ec->cfp); + } + return result; } } #endif @@ -931,7 +941,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (cfp->iseq) { + if (rb_zjit_cfp_has_iseq(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1109,13 +1119,14 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } else { - local_size = ISEQ_BODY(cfp->iseq)->local_table_size; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int ci_offset = local_size - ISEQ_BODY(cfp->iseq)->param.size + VM_ENV_DATA_SIZE; + local_size = ISEQ_BODY(iseq)->local_table_size; + if (ISEQ_BODY(iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int ci_offset = local_size - ISEQ_BODY(iseq)->param.size + VM_ENV_DATA_SIZE; CALL_INFO ci = (CALL_INFO)VM_CF_LEP(cfp)[-ci_offset]; local_size += vm_ci_argc(ci); @@ -1127,8 +1138,8 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co // This is done before creating the imemo_env because VM_STACK_ENV_WRITE // below leaves the on-stack ep in a state that is unsafe to GC. if (VM_FRAME_RUBYFRAME_P(cfp)) { - rb_yjit_invalidate_ep_is_bp(cfp->iseq); - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_yjit_invalidate_ep_is_bp(iseq); + rb_zjit_invalidate_no_ep_escape(iseq); } /* @@ -1156,7 +1167,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co env_ep = &env_body[local_size - 1 /* specval */]; env_ep[VM_ENV_DATA_INDEX_ENV] = (VALUE)env; - env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? cfp->iseq : NULL); + env->iseq = (rb_iseq_t *)(VM_FRAME_RUBYFRAME_P(cfp) ? iseq : NULL); env->ep = env_ep; env->env = env_body; env->env_size = env_size; @@ -1678,8 +1689,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, cfp->iseq); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(ruby_level_cfp->iseq)->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, rb_zjit_cfp_iseq(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(rb_zjit_cfp_iseq(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1986,9 +1997,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (cfp->pc == 0 || cfp->iseq == 0) { + while (!rb_zjit_cfp_has_pc(cfp) || !rb_zjit_cfp_has_iseq(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)cfp->iseq; + struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_zjit_cfp_iseq(cfp); return ifunc->svar_lep; } else { @@ -2071,7 +2082,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(cfp->iseq)); + return RSTRING_PTR(rb_iseq_path(rb_zjit_cfp_iseq(cfp))); } else { return 0; @@ -2100,7 +2111,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(cfp->iseq); + return rb_iseq_path(rb_zjit_cfp_iseq(cfp)); } else { if (pline) *pline = 0; @@ -2582,7 +2593,7 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(ec->cfp); rb_hook_list_t *local_hooks = NULL; unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; if (RB_UNLIKELY(local_hooks_cnt > 0)) { @@ -2835,6 +2846,32 @@ vm_exec_loop(rb_execution_context_t *ec, enum ruby_tag_type state, return result; } +static inline void +zjit_materialize_frames(rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return; + + while (true) { + if (CFP_JIT_RETURN(cfp)) { + const zjit_jit_frame_t *jit_frame = (const zjit_jit_frame_t *)cfp->jit_return; + cfp->pc = jit_frame->pc; + cfp->iseq = (rb_iseq_t *)jit_frame->iseq; + if (jit_frame->materialize_block_code) { + cfp->block_code = NULL; + } + cfp->jit_return = 0; + } + if (VM_FRAME_FINISHED_P(cfp)) break; + cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); + } +} + +void +rb_zjit_materialize_frames(rb_control_frame_t *cfp) +{ + zjit_materialize_frames(cfp); +} + static inline VALUE vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, VALUE errinfo) { @@ -2852,7 +2889,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (ec->cfp->pc == 0 || ec->cfp->iseq == 0) { + while (rb_zjit_cfp_pc(ec->cfp) == 0 || rb_zjit_cfp_iseq(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2866,7 +2903,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + epc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2879,7 +2916,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2906,13 +2943,14 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* TAG_BREAK */ *cfp->sp++ = THROW_DATA_VAL(err); ec->errinfo = Qnil; + zjit_materialize_frames(cfp); return Qundef; } } } if (state == TAG_RAISE) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2928,7 +2966,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2943,7 +2981,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const rb_control_frame_t *escape_cfp; escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -2961,7 +3000,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -2973,7 +3012,8 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V break; } else if (entry->type == type) { - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + entry->cont; + zjit_materialize_frames(cfp); + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -2987,7 +3027,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(cfp->iseq)->catch_table; + ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -3007,8 +3047,9 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); + zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(cfp->iseq)->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -3630,8 +3671,22 @@ rb_execution_context_update(rb_execution_context_t *ec) while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); - cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); - cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + zjit_jit_frame_t *jit_frame = (zjit_jit_frame_t *)cfp->jit_return; + if (jit_frame->iseq) { + // ISEQ frame with JITFrame: relocate iseq in JITFrame + jit_frame->iseq = (const rb_iseq_t *)rb_gc_location((VALUE)jit_frame->iseq); + } + // block_code must always be relocated. For ISEQ frames, the JIT caller + // may have written it (gen_block_handler_specval) for passing blocks. + // For C frames, rb_iterate0 may have written an ifunc to block_code + // after the JIT pushed the frame. NULL is safe to pass to rb_gc_location. + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } + else { + cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); + cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); + } if (!VM_ENV_LOCAL_P(ep)) { const VALUE *prev_ep = VM_ENV_PREV_EP(ep); @@ -3682,7 +3737,9 @@ rb_execution_context_mark(const rb_execution_context_t *ec) VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)cfp->iseq); + rb_gc_mark_movable((VALUE)rb_zjit_cfp_iseq(cfp)); + // Mark block_code directly (not through rb_zjit_cfp_block_code) + // because rb_iterate0 may write a valid ifunc after JIT frame push. rb_gc_mark_movable((VALUE)cfp->block_code); if (VM_ENV_LOCAL_P(ep) && VM_ENV_BOXED_P(ep)) { diff --git a/vm_backtrace.c b/vm_backtrace.c index 07d2e33e321787..674cfe24274c24 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -18,6 +18,7 @@ #include "ruby/debug.h" #include "ruby/encoding.h" #include "vm_core.h" +#include "zjit.h" static VALUE rb_cBacktrace; static VALUE rb_cBacktraceLocation; @@ -101,9 +102,9 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && cfp->iseq) { - const rb_iseq_t *iseq = cfp->iseq; - int line = calc_lineno(iseq, cfp->pc); + if (VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_iseq(cfp)) { + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + int line = calc_lineno(iseq, rb_zjit_cfp_pc(cfp)); if (line != 0) { return line; } @@ -617,7 +618,7 @@ backtrace_size(const rb_execution_context_t *ec) static bool is_rescue_or_ensure_frame(const rb_control_frame_t *cfp) { - enum rb_iseq_type type = ISEQ_BODY(cfp->iseq)->type; + enum rb_iseq_type type = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type; return type == ISEQ_TYPE_RESCUE || type == ISEQ_TYPE_ENSURE; } @@ -687,17 +688,17 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq) { - if (cfp->pc) { + if (rb_zjit_cfp_has_iseq(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { if (start_frame > 0) { start_frame--; } else { - bool internal = is_internal_location(cfp->iseq); + bool internal = is_internal_location(rb_zjit_cfp_iseq(cfp)); if (skip_internal && internal) continue; if (!skip_next_frame) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const VALUE *pc = rb_zjit_cfp_pc(cfp); if (internal && backpatch_counter > 0) { // To keep only one internal frame, discard the previous backpatch frames bt->backtrace_size -= backpatch_counter; @@ -752,10 +753,10 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (cfp->iseq && cfp->pc && !(skip_internal && is_internal_location(cfp->iseq))) { + if (rb_zjit_cfp_has_iseq(cfp) && rb_zjit_cfp_has_pc(cfp) && !(skip_internal && is_internal_location(rb_zjit_cfp_iseq(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc - bt_backpatch_loc(backpatch_counter, loc, cfp->iseq, cfp->pc); - RB_OBJ_WRITTEN(btobj, Qundef, cfp->iseq); + bt_backpatch_loc(backpatch_counter, loc, rb_zjit_cfp_iseq(cfp), rb_zjit_cfp_pc(cfp)); + RB_OBJ_WRITTEN(btobj, Qundef, rb_zjit_cfp_iseq(cfp)); if (do_yield) { bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); } @@ -1019,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (cfp->iseq) { - if (cfp->pc) { + if (rb_zjit_cfp_has_iseq(cfp)) { + if (rb_zjit_cfp_has_pc(cfp)) { iter_iseq(arg, cfp); } } @@ -1052,8 +1053,8 @@ oldbt_init(void *ptr, size_t dmy) static void oldbt_iter_iseq(void *ptr, const rb_control_frame_t *cfp) { - const rb_iseq_t *iseq = cfp->iseq; - const VALUE *pc = cfp->pc; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const VALUE *pc = rb_zjit_cfp_pc(cfp); struct oldbt_arg *arg = (struct oldbt_arg *)ptr; VALUE file = arg->filename = rb_iseq_path(iseq); VALUE name = ISEQ_BODY(iseq)->location.label; @@ -1550,17 +1551,18 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) { struct collect_caller_bindings_data *data = (struct collect_caller_bindings_data *)arg; VALUE frame = rb_ary_new2(6); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); rb_ary_store(frame, CALLER_BINDING_SELF, cfp->self); rb_ary_store(frame, CALLER_BINDING_CLASS, get_klass(cfp)); rb_ary_store(frame, CALLER_BINDING_BINDING, GC_GUARDED_PTR(cfp)); /* create later */ - rb_ary_store(frame, CALLER_BINDING_ISEQ, cfp->iseq ? (VALUE)cfp->iseq : Qnil); + rb_ary_store(frame, CALLER_BINDING_ISEQ, iseq ? (VALUE)iseq : Qnil); rb_ary_store(frame, CALLER_BINDING_CFP, GC_GUARDED_PTR(cfp)); rb_backtrace_location_t *loc = &data->bt->backtrace[data->bt->backtrace_size++]; RB_OBJ_WRITE(data->btobj, &loc->cme, rb_vm_frame_method_entry(cfp)); - RB_OBJ_WRITE(data->btobj, &loc->iseq, cfp->iseq); - loc->pc = cfp->pc; + RB_OBJ_WRITE(data->btobj, &loc->iseq, iseq); + loc->pc = rb_zjit_cfp_pc(cfp); VALUE vloc = location_create(loc, (void *)data->btobj); rb_ary_store(frame, CALLER_BINDING_LOC, vloc); @@ -1745,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; ipc != 0) { + if (VM_FRAME_RUBYFRAME_P_UNCHECKED(cfp) && rb_zjit_cfp_has_pc(cfp)) { if (start > 0) { start--; continue; @@ -1753,17 +1755,18 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b /* record frame info */ cme = rb_vm_frame_method_entry_unchecked(cfp); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); if (cme && cme->def->type == VM_METHOD_TYPE_ISEQ) { buff[i] = (VALUE)cme; } else { - buff[i] = (VALUE)cfp->iseq; + buff[i] = (VALUE)iseq; } if (lines) { - const VALUE *pc = cfp->pc; - VALUE *iseq_encoded = ISEQ_BODY(cfp->iseq)->iseq_encoded; - VALUE *pc_end = iseq_encoded + ISEQ_BODY(cfp->iseq)->iseq_size; + const VALUE *pc = rb_zjit_cfp_pc(cfp); + VALUE *iseq_encoded = ISEQ_BODY(iseq)->iseq_encoded; + VALUE *pc_end = iseq_encoded + ISEQ_BODY(iseq)->iseq_size; // The topmost frame may have an invalid PC because the JIT // may leave it uninitialized for speed. JIT code must update the PC @@ -1778,7 +1781,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b lines[i] = 0; } else { - lines[i] = calc_lineno(cfp->iseq, pc); + lines[i] = calc_lineno(iseq, pc); } } diff --git a/vm_core.h b/vm_core.h index 85711ffc7e8113..3bf1d51b0921ce 100644 --- a/vm_core.h +++ b/vm_core.h @@ -1532,7 +1532,10 @@ static inline int VM_FRAME_CFRAME_P(const rb_control_frame_t *cfp) { int cframe_p = VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_CFRAME) != 0; - VM_ASSERT(RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || + // With ZJIT lightweight frames, cfp->iseq may be stale (not yet materialized), + // so skip this assertion when jit_return is set (zjit.h is not available here). + VM_ASSERT(cfp->jit_return || + RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || (VM_FRAME_TYPE(cfp) & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY); return cframe_p; } diff --git a/vm_dump.c b/vm_dump.c index 4864ccf58bdb30..8e92566ce347ad 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -37,6 +37,7 @@ #include "iseq.h" #include "vm_core.h" #include "ractor_core.h" +#include "zjit.h" #define MAX_POSBUF 128 @@ -118,21 +119,21 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (cfp->iseq != 0) { + if (rb_zjit_cfp_has_iseq(cfp)) { + iseq = rb_zjit_cfp_iseq(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + if (RUBY_VM_IFUNC_P(iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)iseq)) { + tmp = rb_sym2str((VALUE)iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (rb_zjit_cfp_has_pc(cfp)) { + pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -169,6 +170,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c else { kprintf("b:---- "); } + kprintf("r:%p ", cfp->jit_return); kprintf("%-6s", magic); if (line) { kprintf(" %s", posbuf); @@ -339,21 +341,22 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && cfp->iseq != 0) { + if (cfp && rb_zjit_cfp_has_iseq(cfp)) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - if (RUBY_VM_IFUNC_P(cfp->iseq)) { + const rb_iseq_t *resolved_iseq = rb_zjit_cfp_iseq(cfp); + if (RUBY_VM_IFUNC_P(resolved_iseq)) { iseq_name = ""; } - else if (SYMBOL_P((VALUE)cfp->iseq)) { - tmp = rb_sym2str((VALUE)cfp->iseq); + else if (SYMBOL_P((VALUE)resolved_iseq)) { + tmp = rb_sym2str((VALUE)resolved_iseq); iseq_name = RSTRING_PTR(tmp); snprintf(posbuf, MAX_POSBUF, ":%s", iseq_name); line = -1; } else { - if (cfp->pc) { - iseq = cfp->iseq; - pc = cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + if (rb_zjit_cfp_has_pc(cfp)) { + iseq = resolved_iseq; + pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); diff --git a/vm_eval.c b/vm_eval.c index 25d366f5cd19e6..02fd83ef699923 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1988,12 +1988,12 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li block.as.captured = *VM_CFP_TO_CAPTURED_BLOCK(cfp); block.as.captured.self = self; - block.as.captured.code.iseq = cfp->iseq; + block.as.captured.code.iseq = rb_zjit_cfp_iseq(cfp); block.type = block_type_iseq; // EP is not escaped to the heap here, but captured and reused by another frame. // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT. - rb_zjit_invalidate_no_ep_escape(cfp->iseq); + rb_zjit_invalidate_no_ep_escape(rb_zjit_cfp_iseq(cfp)); iseq = eval_make_iseq(src, file, line, &block); if (!iseq) { @@ -2849,10 +2849,11 @@ rb_current_realfilepath(void) rb_control_frame_t *cfp = ec->cfp; cfp = vm_get_ruby_level_caller_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)); if (cfp != NULL) { - VALUE path = rb_iseq_realpath(cfp->iseq); + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + VALUE path = rb_iseq_realpath(iseq); if (RTEST(path)) return path; // eval context - path = rb_iseq_path(cfp->iseq); + path = rb_iseq_path(iseq); if (path == eval_default_path) { return Qnil; } diff --git a/vm_exec.h b/vm_exec.h index 641ace4eaf29b9..340fadff517223 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -189,6 +189,10 @@ default: \ rb_jit_func_t func = zjit_compile(ec); \ if (func) { \ val = zjit_entry(ec, ec->cfp, func); \ + if (UNDEF_P(val)) { \ + ec->cfp->jit_return = 0; \ + zjit_materialize_frames(ec->cfp); \ + } \ } \ } \ } \ diff --git a/vm_insnhelper.c b/vm_insnhelper.c index ba7523eb285d07..1b5a40bc9768af 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1767,16 +1767,16 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c escape_cfp = reg_cfp; while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) { - if (ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + if (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp); ep = escape_cfp->ep; - base_iseq = escape_cfp->iseq; + base_iseq = rb_zjit_cfp_iseq(escape_cfp); } else { ep = VM_ENV_PREV_EP(ep); base_iseq = ISEQ_BODY(base_iseq)->parent_iseq; escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep); - VM_ASSERT(escape_cfp->iseq == base_iseq); + VM_ASSERT(rb_zjit_cfp_iseq(escape_cfp) == base_iseq); } } @@ -1790,8 +1790,8 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c while (escape_cfp < eocfp) { if (escape_cfp->ep == ep) { - const rb_iseq_t *const iseq = escape_cfp->iseq; - const VALUE epc = escape_cfp->pc - ISEQ_BODY(iseq)->iseq_encoded; + const rb_iseq_t *const iseq = rb_zjit_cfp_iseq(escape_cfp); + const VALUE epc = rb_zjit_cfp_pc(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table; unsigned int i; @@ -1850,7 +1850,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c if (lep == target_lep && VM_FRAME_RUBYFRAME_P(escape_cfp) && - ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) { + ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { in_class_frame = 1; target_lep = 0; } @@ -1880,7 +1880,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) { - switch (ISEQ_BODY(escape_cfp->iseq)->type) { + switch (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type) { case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: if (toplevel) { @@ -1894,7 +1894,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } break; case ISEQ_TYPE_EVAL: { - const rb_iseq_t *is = escape_cfp->iseq; + const rb_iseq_t *is = rb_zjit_cfp_iseq(escape_cfp); enum rb_iseq_type t = ISEQ_BODY(is)->type; while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) { if (!(is = ISEQ_BODY(is)->parent_iseq)) break; @@ -1912,7 +1912,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } - if (escape_cfp->ep == target_lep && ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_METHOD) { + if (escape_cfp->ep == target_lep && ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_METHOD) { if (target_ep == NULL) { goto valid_return; } @@ -2376,7 +2376,7 @@ vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct } #endif - return vm_search_method_slowpath0((VALUE)reg_cfp->iseq, cd, klass); + return vm_search_method_slowpath0((VALUE)rb_zjit_cfp_iseq(reg_cfp), cd, klass); } static const struct rb_callable_method_entry_struct * @@ -4705,7 +4705,7 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c if (ref_cme) { if (calling->cd->cc) { const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); - RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(cfp), &calling->cd->cc, cc); return vm_call_method(ec, cfp, calling); } else { @@ -5126,6 +5126,7 @@ static const struct rb_callcache * vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE current_defined_class; + const rb_iseq_t *iseq = rb_zjit_cfp_iseq(reg_cfp); const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp); if (!me) { @@ -5135,7 +5136,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c current_defined_class = vm_defined_class_for_protected_call(me); if (BUILTIN_TYPE(current_defined_class) != T_MODULE && - reg_cfp->iseq != method_entry_iseqptr(me) && + iseq != method_entry_iseqptr(me) && !rb_obj_is_kind_of(recv, current_defined_class)) { VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ? RCLASS_INCLUDER(current_defined_class) : current_defined_class; @@ -5167,7 +5168,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c vm_ci_argc(cd->ci), vm_ci_kwarg(cd->ci)); - RB_OBJ_WRITTEN(reg_cfp->iseq, Qundef, cd->ci); + RB_OBJ_WRITTEN(iseq, Qundef, cd->ci); } const struct rb_callcache *cc; @@ -5177,7 +5178,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c if (!klass) { /* bound instance method of module */ cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { cc = vm_search_method_fastpath(reg_cfp, cd, klass); @@ -5192,7 +5193,7 @@ vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *c const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid); if (cme) { cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super); - RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc); + RB_OBJ_WRITE(iseq, &cd->cc, cc); } else { cd->cc = cc = empty_cc_for_super(); @@ -6155,6 +6156,7 @@ rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6168,7 +6170,7 @@ rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6197,6 +6199,7 @@ rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ return val; } +// Fallback for YJIT/ZJIT, not used by the interpreter VALUE rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq) { @@ -6209,7 +6212,7 @@ rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6617,7 +6620,7 @@ rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const vm_ic_track_const_chain(GET_CFP(), ic, segments); // Undo the PC increment to get the address to this instruction // INSN_ATTR(width) == 2 - vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2); + vm_ic_update(rb_zjit_cfp_iseq(GET_CFP()), ic, val, GET_EP(), rb_zjit_cfp_pc(GET_CFP()) - 2); } return val; } diff --git a/yjit.c b/yjit.c index 2e7216a1915406..46565cb6c0d035 100644 --- a/yjit.c +++ b/yjit.c @@ -480,7 +480,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // If it's a FINISH frame, just normally exit with a non-Qundef value. cfp->jit_return = leave_exit; } - else if (cfp->jit_return) { + else if (CFP_JIT_RETURN(cfp)) { while (!VM_FRAME_FINISHED_P(cfp)) { if (cfp->jit_return == leave_exit) { // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on diff --git a/zjit.h b/zjit.h index d1d1b01df3b68e..e0cdf985f133e0 100644 --- a/zjit.h +++ b/zjit.h @@ -9,6 +9,14 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif +// JITFrame is a C ABI compatible struct defined in Rust with #[repr(C)]. +// C code can read its fields directly without calling Rust accessor functions. +typedef struct zjit_jit_frame { + const VALUE *pc; + const rb_iseq_t *iseq; // marked in rb_execution_context_mark + bool materialize_block_code; +} zjit_jit_frame_t; + #if USE_ZJIT extern void *rb_zjit_entry; extern uint64_t rb_zjit_call_threshold; @@ -46,4 +54,64 @@ static inline void rb_zjit_invalidate_root_box(void) {} #define rb_zjit_enabled_p (rb_zjit_entry != 0) +enum zjit_poison_values { + // Poison value used on frame push when runtime checks are enabled + ZJIT_JIT_RETURN_POISON = 2, +}; + +// Check if cfp->jit_return holds a ZJIT lightweight frame (JITFrame pointer). +// YJIT also uses jit_return (as a return address), so this must only return +// true when ZJIT is enabled and has set jit_return to a JITFrame pointer. +static inline bool +CFP_JIT_RETURN(const rb_control_frame_t *cfp) +{ + if (!rb_zjit_enabled_p) return false; +#if USE_ZJIT + RUBY_ASSERT_ALWAYS(cfp->jit_return != (void *)ZJIT_JIT_RETURN_POISON); +#endif + return !!cfp->jit_return; +} + +// Returns true if cfp has an ISEQ, either directly or via JITFrame. +// When JITFrame is present, it is authoritative (cfp->iseq may be stale). +// C frames with JITFrame have iseq=NULL, so this returns false for them. +static inline bool +rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq != NULL; + return !!cfp->iseq; +} + +// Returns true if cfp has a PC, either directly or via JITFrame. +// When JITFrame is present, it is authoritative (cfp->pc may be stale/poisoned). +// C frames with JITFrame have pc=NULL, so this returns false for them. +static inline bool +rb_zjit_cfp_has_pc(const rb_control_frame_t *cfp) +{ + if (CFP_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->pc != NULL; + return !!cfp->pc; +} + +static inline const VALUE* +rb_zjit_cfp_pc(const rb_control_frame_t *cfp) +{ + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; + } + else { + return cfp->pc; + } +} + +static inline const rb_iseq_t* +rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) +{ + if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; + } + else { + return cfp->iseq; + } +} + #endif // #ifndef ZJIT_H diff --git a/zjit.rb b/zjit.rb index e298b509049039..ffee0006962fc9 100644 --- a/zjit.rb +++ b/zjit.rb @@ -141,6 +141,7 @@ def stats_string ], buf:, stats:, right_align: true, base: :send_count) print_counters([ :compiled_iseq_count, + :compiled_side_exit_count, :failed_iseq_count, :compile_time_ns, diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 34720e77eb7ebe..33dd4eac1e1311 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -256,8 +256,7 @@ fn main() { .allowlist_type("iseq_inline_cvar_cache_entry") .blocklist_type("rb_execution_context_.*") // Large struct with various-type fields and an ifdef, so we don't import .opaque_type("rb_execution_context_.*") - .blocklist_type("rb_control_frame_struct") - .opaque_type("rb_control_frame_struct") + .allowlist_type("rb_control_frame_struct") .allowlist_function("rb_vm_bh_to_procval") .allowlist_function("rb_vm_env_write") .allowlist_function("rb_vm_ep_local_ep") @@ -310,6 +309,7 @@ fn main() { .allowlist_function("rb_zjit_insn_leaf") .allowlist_type("jit_bindgen_constants") .allowlist_type("zjit_struct_offsets") + .allowlist_type("zjit_poison_values") .allowlist_function("rb_assert_holding_vm_lock") .allowlist_function("rb_jit_shape_too_complex_p") .allowlist_function("rb_jit_multi_ractor_p") diff --git a/zjit/src/backend/arm64/mod.rs b/zjit/src/backend/arm64/mod.rs index 9083824c4b28af..ed126cdf284efa 100644 --- a/zjit/src/backend/arm64/mod.rs +++ b/zjit/src/backend/arm64/mod.rs @@ -1754,7 +1754,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 76b11f26b2613b..624fcd0389f6f2 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -6,7 +6,7 @@ use std::rc::Rc; use std::sync::{Arc, Mutex}; use crate::bitset::BitSet; use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end}; -use crate::cruby::{Qundef, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; +use crate::cruby::{IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary}; use crate::hir::{Invariant, SideExitReason}; use crate::hir; use crate::options::{TraceExits, PerfMap, get_option}; @@ -551,6 +551,7 @@ pub struct SideExit { pub pc: Opnd, pub stack: Vec, pub locals: Vec, + pub iseq: IseqPtr, /// If set, the side exit will call the recompile function with these arguments /// to profile the send and invalidate the ISEQ for recompilation. pub recompile: Option, @@ -2619,7 +2620,7 @@ impl Assembler pub fn compile_exits(&mut self) -> Vec { /// Restore VM state (cfp->pc, cfp->sp, stack, locals) for the side exit. fn compile_exit_save_state(asm: &mut Assembler, exit: &SideExit) { - let SideExit { pc, stack, locals, .. } = exit; + let SideExit { pc, stack, locals, iseq, .. } = exit; // Side exit blocks are not part of the CFG at the moment, // so we need to manually ensure that patchpoints get padded @@ -2632,6 +2633,11 @@ impl Assembler asm_comment!(asm, "save cfp->sp"); asm.lea_into(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_SP), Opnd::mem(64, SP, stack.len() as i32 * SIZEOF_VALUE_I32)); + asm_comment!(asm, "save cfp->iseq"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into()); + + // cfp->block_code and cfp->jit_return are cleared by the caller jit_exec() or JIT_EXEC() + if !stack.is_empty() { asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", ")); for (idx, &opnd) in stack.iter().enumerate() { @@ -2663,6 +2669,16 @@ impl Assembler // compile_exit_save_state because it clobbers caller-saved registers // that may hold stack/local operands we need to save. if let Some(recompile) = &exit.recompile { + if cfg!(feature = "runtime_checks") { + // Clear jit_return to fully materialize the frame. This must happen + // before any C call in the exit path (e.g. no_profile_send_recompile) + // because that C call can trigger GC, which walks the stack and would + // hit the CFP_JIT_RETURN assertion if jit_return still holds the + // runtime_checks poison value (JIT_RETURN_POISON). + asm_comment!(asm, "clear cfp->jit_return"); + asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); + } + use crate::codegen::no_profile_send_recompile; asm_comment!(asm, "profile and maybe recompile for no-profile send"); asm_ccall!(asm, no_profile_send_recompile, @@ -2791,6 +2807,7 @@ impl Assembler if !compiled_exits.is_empty() { let nanos = side_exit_start.elapsed().as_nanos(); crate::stats::incr_counter_by(crate::stats::Counter::compile_side_exit_time_ns, nanos as u64); + crate::stats::incr_counter_by(crate::stats::Counter::compiled_side_exit_count, compiled_exits.len() as u64); } // Close the current perf range for side exits diff --git a/zjit/src/backend/x86_64/mod.rs b/zjit/src/backend/x86_64/mod.rs index 3cf744990d8381..f7900f025ee0a1 100644 --- a/zjit/src/backend/x86_64/mod.rs +++ b/zjit/src/backend/x86_64/mod.rs @@ -1388,7 +1388,7 @@ mod tests { let val64 = asm.add(CFP, Opnd::UImm(64)); asm.store(Opnd::mem(64, SP, 0x10), val64); - let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: Opnd::const_ptr(0 as *const u8), stack: vec![], locals: vec![], recompile: None } }; + let side_exit = Target::SideExit { reason: SideExitReason::Interrupt, exit: SideExit { pc: 0.into(), iseq: std::ptr::null(), stack: vec![], locals: vec![], recompile: None } }; asm.push_insn(Insn::Joz(val64, side_exit)); asm.mov(C_ARG_OPNDS[0], C_RET_OPND.with_num_bits(32)); asm.mov(C_ARG_OPNDS[1], Opnd::mem(64, SP, -8)); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index b21d4070d2a222..260351b595bd8e 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -14,7 +14,7 @@ use crate::invariants::{ track_root_box_assumption }; use crate::gc::append_gc_offsets; -use crate::payload::{get_or_create_iseq_payload, IseqCodePtrs, IseqVersion, IseqVersionRef, IseqStatus}; +use crate::payload::{IseqCodePtrs, IseqStatus, IseqVersion, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::state::ZJITState; use crate::stats::{CompileError, exit_counter_for_compile_error, exit_counter_for_unhandled_hir_insn, incr_counter, incr_counter_by, send_fallback_counter, send_fallback_counter_for_method_type, send_fallback_counter_for_super_method_type, send_fallback_counter_ptr_for_opcode, send_without_block_fallback_counter_for_method_type, send_without_block_fallback_counter_for_optimized_method_type}; use crate::stats::{counter_ptr, with_time_stat, Counter, Counter::{compile_time_ns, exit_compile_error}}; @@ -36,6 +36,13 @@ const PC_POISON: Option<*const VALUE> = if cfg!(feature = "runtime_checks") { None }; +/// Sentinel jit_return stored on ISEQ frame push when runtime checks are enabled. +const JIT_RETURN_POISON: Option = if cfg!(feature = "runtime_checks") { + Some(ZJIT_JIT_RETURN_POISON as usize) +} else { + None +}; + /// Ephemeral code generation state struct JITState { /// Instruction sequence for the method being compiled @@ -1045,8 +1052,8 @@ fn gen_ccall_with_frame( iseq: None, cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, - pc: PC_POISON, specval: block_handler_specval, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1135,7 +1142,7 @@ fn gen_ccall_variadic( cme, frame_type: VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL, specval: block_handler_specval, - pc: PC_POISON, + write_block_code: false, }); asm_comment!(asm, "switch to new SP register"); @@ -1591,8 +1598,8 @@ fn gen_send_iseq_direct( iseq: Some(iseq), cme, frame_type, - pc: None, specval, + write_block_code: iseq_may_write_block_code(iseq), }); // Write "keyword_bits" to the callee's frame if the callee accepts keywords. @@ -2696,6 +2703,35 @@ fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReaso } } +/// Check if an ISEQ contains instructions that may write to block_code +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock, and their trace variants). +/// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. +#[allow(non_upper_case_globals)] +fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { + let encoded_size = unsafe { rb_iseq_encoded_size(iseq) }; + let mut insn_idx: u32 = 0; + + while insn_idx < encoded_size { + let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; + let opcode = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } as u32; + + match opcode { + YARVINSN_send | YARVINSN_trace_send | + YARVINSN_sendforward | YARVINSN_trace_sendforward | + YARVINSN_invokesuper | YARVINSN_trace_invokesuper | + YARVINSN_invokesuperforward | YARVINSN_trace_invokesuperforward | + YARVINSN_invokeblock | YARVINSN_trace_invokeblock => { + return true; + } + _ => {} + } + + insn_idx = insn_idx.saturating_add(unsafe { rb_insn_len(VALUE(opcode as usize)) }.try_into().unwrap()); + } + + false +} + /// Save only the PC to CFP. Use this when you need to call gen_save_sp() /// immediately after with a custom stack size (e.g., gen_ccall_with_frame /// adjusts SP to exclude receiver and arguments). @@ -2705,7 +2741,11 @@ fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { gen_incr_counter(asm, Counter::vm_write_pc_count); asm_comment!(asm, "save PC to CFP"); - asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(next_pc)); + if let Some(pc) = PC_POISON { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + } + let jit_frame = JITFrame::new(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } /// Save the current PC on the CFP as a preparation for calling a C function @@ -2800,7 +2840,9 @@ struct ControlFrame { /// The [`VM_ENV_DATA_INDEX_SPECVAL`] slot of the frame. /// For the type of frames we push, block handler or the parent EP. specval: lir::Opnd, - pc: Option<*const VALUE>, + /// Whether to write block_code = 0 at frame push time. + /// True when the callee ISEQ may write to block_code (has send/invokesuper/invokeblock). + write_block_code: bool, } /// Compile an interpreter frame @@ -2830,25 +2872,35 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C asm_comment!(asm, "push callee control frame"); - if let Some(iseq) = frame.iseq { - // cfp_opnd(RUBY_OFFSET_CFP_PC): written by the callee frame on side-exits, non-leaf calls, or calls with GC - // cfp_opnd(RUBY_OFFSET_CFP_SP): written by the callee frame on side-exits, non-leaf calls, or calls with GC - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), VALUE::from(iseq).into()); + if frame.iseq.is_some() { + // PC, SP, and ISEQ are written lazily by the callee on side-exits, non-leaf calls, or GC. + if let Some(jit_return_poison) = JIT_RETURN_POISON { + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), jit_return_poison.into()); + } + if frame.write_block_code { + asm_comment!(asm, "write block_code for iseq that may use it"); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + } } else { - // C frames don't have a PC and ISEQ in normal operation. - // When runtime checks are enabled we poison the PC so accidental reads stand out. - if let Some(pc) = frame.pc { - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); + // C frames don't have a PC and ISEQ in normal operation. ISEQ frames set PC on gen_save_pc_for_gc(). + // When runtime checks are enabled we poison the PC for C frames so accidental reads stand out. + if let (None, Some(pc)) = (frame.iseq, PC_POISON) { + asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); } let new_sp = asm.lea(Opnd::mem(64, SP, (ep_offset + 1) * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SP), new_sp); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_ISEQ), 0.into()); + // block_code must be written explicitly because the interpreter reads + // captured->code.ifunc directly from cfp->block_code (not through JITFrame). + // Without this, stale data from a previous frame occupying this CFP slot + // can be used as an ifunc pointer, causing a segfault. + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); + let jit_frame = JITFrame::new(std::ptr::null(), std::ptr::null(), false); + asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } asm.mov(cfp_opnd(RUBY_OFFSET_CFP_SELF), frame.recv); let ep = asm.lea(Opnd::mem(64, SP, ep_offset * SIZEOF_VALUE_I32)); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_EP), ep); - asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); } /// Stack overflow check: fails if CFP<=SP at any point in the callee. @@ -2932,6 +2984,7 @@ fn build_side_exit(jit: &JITState, state: &FrameState) -> SideExit { pc: Opnd::const_ptr(state.pc), stack, locals, + iseq: jit.iseq, recompile: None, } } @@ -3010,19 +3063,32 @@ c_callable! { /// We should be able to compile most (if not all) function stubs by side-exiting at unsupported /// instructions, so this should be used primarily for cb.has_dropped_bytes() situations. fn function_stub_hit(iseq_call_ptr: *const c_void, cfp: CfpPtr, sp: *mut VALUE, ec: EcPtr) -> *const u8 { - with_vm_lock(src_loc!(), || { - // gen_push_frame() doesn't set PC, so we need to set them before exit. - // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + // Make sure cfp is ready to be scanned by other Ractors and GC before taking the barrier + { + unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); } let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; let iseq = iseq_call.iseq.get(); let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); + // gen_push_frame() doesn't set PC, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; + unsafe { (*cfp).iseq = iseq }; + } + + with_vm_lock(src_loc!(), || { + // Re-create the Rc inside the VM lock because IseqCall's interior + // mutability (Cell) requires exclusive access. + let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; + let iseq = iseq_call.iseq.get(); // JIT-to-JIT calls don't eagerly fill nils to non-parameter locals. // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here. fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, compile_error: &CompileError) { unsafe { + // Caller frames are materialized by jit_exec() after the entry trampoline returns. + // The current frame's pc and iseq are already set by function_stub_hit before this point. + // Set SP which gen_push_frame() doesn't set rb_set_cfp_sp(cfp, sp); diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index 6ca9a97b83865e..93079a6b5492da 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -373,13 +373,6 @@ pub struct rb_callcache { _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, } -/// Opaque control_frame (CFP) struct from vm_core.h -#[repr(C)] -pub struct rb_control_frame_struct { - _data: [u8; 0], - _marker: core::marker::PhantomData<(*mut u8, core::marker::PhantomPinned)>, -} - /// Pointer to a control frame pointer (CFP) pub type CfpPtr = *mut rb_control_frame_struct; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 5c7ce49fc612da..b270929da7c31e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1299,6 +1299,16 @@ pub struct rb_block__bindgen_ty_1 { pub proc_: __BindgenUnionField, pub bindgen_union_field: [u64; 3usize], } +#[repr(C)] +pub struct rb_control_frame_struct { + pub pc: *const VALUE, + pub sp: *mut VALUE, + pub iseq: *const rb_iseq_t, + pub self_: VALUE, + pub ep: *const VALUE, + pub block_code: *const ::std::os::raw::c_void, + pub jit_return: *mut ::std::os::raw::c_void, +} pub type rb_control_frame_t = rb_control_frame_struct; #[repr(C)] pub struct rb_proc_t { @@ -1900,6 +1910,8 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; +pub const ZJIT_JIT_RETURN_POISON: zjit_poison_values = 2; +pub type zjit_poison_values = u32; pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; pub type zjit_struct_offsets = u32; pub const ROBJECT_OFFSET_AS_HEAP_FIELDS: jit_bindgen_constants = 16; diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index b79f19837ff09a..1a0990bc6d5a3a 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -3,7 +3,7 @@ use std::ptr::null; use std::{ffi::c_void, ops::Range}; use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; -use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; +use crate::payload::{IseqPayload, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; use crate::stats::Counter::gc_time_ns; /// GC callback for marking GC objects in the per-ISEQ payload. @@ -89,6 +89,20 @@ pub extern "C" fn rb_zjit_root_update_references() { } let invariants = ZJITState::get_invariants(); invariants.update_references(); + + // Update iseq pointers in all JITFrames for GC compaction. + // rb_execution_context_update only updates JITFrames currently on the stack, + // but JITFrames not on the stack also need their iseq pointers updated + // because the JIT code will reuse them on the next call. + for &jit_frame in ZJITState::get_jit_frames().iter() { + let old_iseq = unsafe { (*jit_frame).iseq }; + if !old_iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(old_iseq)) }.as_iseq(); + if old_iseq != new_iseq { + unsafe { (*(jit_frame as *mut JITFrame)).iseq = new_iseq; } + } + } + } } fn iseq_mark(payload: &IseqPayload) { @@ -206,5 +220,15 @@ fn ranges_overlap(left: &Range, right: &Range) -> bool where T: Partial /// Callback for marking GC objects inside [crate::invariants::Invariants]. #[unsafe(no_mangle)] pub extern "C" fn rb_zjit_root_mark() { - // TODO(max): Either add roots to mark or consider removing this callback + // Mark iseq pointers in all JITFrames. JITFrames that are currently on the + // stack are also marked via rb_execution_context_mark, but JITFrames not on + // the stack still need their iseqs kept alive because JIT code will reuse them. + if !ZJITState::has_instance() { + return; + } + for &jit_frame in ZJITState::get_jit_frames().iter() { + if !unsafe { (*jit_frame).iseq }.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from((*jit_frame).iseq)); } + } + } } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 097e1b824be664..b0a53ec56493f4 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -6520,7 +6520,7 @@ impl<'a> std::fmt::Display for FunctionGraphvizPrinter<'a> { #[derive(Debug, Clone, PartialEq)] pub struct FrameState { - iseq: IseqPtr, + pub iseq: IseqPtr, insn_idx: usize, // Ruby bytecode instruction pointer pub pc: *const VALUE, diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs new file mode 100644 index 00000000000000..899d567177450a --- /dev/null +++ b/zjit/src/jit_frame.rs @@ -0,0 +1,288 @@ +use crate::cruby::{IseqPtr, VALUE}; +use crate::state::ZJITState; + +#[derive(Debug)] +#[repr(C)] +pub struct JITFrame { + pub pc: *const VALUE, + pub iseq: IseqPtr, // marked in rb_execution_context_mark + pub materialize_block_code: bool, +} + +impl JITFrame { + /// Allocate a new JITFrame on the heap and register it with ZJITState. + /// Returns a raw pointer that remains valid for the lifetime of the process. + pub fn new(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { + let jit_frame = Box::new(JITFrame { pc, iseq, materialize_block_code }); + let raw_ptr = Box::into_raw(jit_frame) as *const _; + ZJITState::get_jit_frames().push(raw_ptr); + raw_ptr + } +} + +#[cfg(test)] +mod tests { + use crate::cruby::{eval, inspect}; + use insta::assert_snapshot; + + #[test] + fn test_jit_frame_entry_first() { + eval(r#" + def test + itself + callee + end + + def callee + caller + end + + test + "#); + assert_snapshot!(inspect("test.first"), @r#"":4:in 'Object#test'""#); + } + + #[test] + fn test_materialize_one_frame() { + assert_snapshot!(inspect(" + def jit_entry + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + #[test] + fn test_materialize_two_frames() { // materialize caller frames on raise + // At the point of `resuce`, there are two lightweight frames on stack and both need to be + // materialized before passing control to interpreter. + assert_snapshot!(inspect(" + def jit_entry = raise_and_rescue + def raise_and_rescue + raise rescue 1 + end + jit_entry + jit_entry + "), @"1"); + } + + // Materialize frames on side exit: a type guard triggers a side exit with + // multiple JIT frames on the stack. All frames must be materialized before + // the interpreter resumes. + #[test] + fn test_side_exit_materialize_frames() { + assert_snapshot!(inspect(" + def side_exit(n) = 1 + n + def jit_frame(n) = 1 + side_exit(n) + def entry(n) = jit_frame(n) + entry(2) + [entry(2), entry(2.0)] + "), @"[4, 4.0]"); + } + + // BOP invalidation must not overwrite the top-most frame's PC with + // jit_frame's PC. After invalidation the interpreter resumes at a new + // PC, so a stale jit_frame PC would cause wrong execution. + #[test] + fn test_bop_invalidation() { + assert_snapshot!(inspect(r#" + def test + eval("class Integer; def +(_) = 100; end") + 1 + 2 + end + test + test + "#), @"100"); + } + + // Side exit at the very start of a method, before any jit_return has been + // written by gen_save_pc_for_gc. The jit_return field should be 0 (from + // vm_push_frame), so materialization should be a no-op for that frame. + #[test] + fn test_side_exit_before_jit_return_write() { + assert_snapshot!(inspect(" + def entry(n) = n + 1 + entry(1) + [entry(1), entry(1.0)] + "), @"[2, 2.0]"); + } + + #[test] + fn test_caller_iseq() { + assert_snapshot!(inspect(r#" + def callee = call_caller + def test = callee + + def callee2 = call_caller + def test2 = callee2 + + def call_caller = caller + + test + test2 + test.first + "#), @r#"":2:in 'Object#callee'""#); + } + + // ISEQ must be readable during exception handling so the interpreter + // can look up rescue/ensure tables. + #[test] + fn test_iseq_on_raise() { + assert_snapshot!(inspect(r#" + def jit_entry(v) = make_range_then_exit(v) + def make_range_then_exit(v) + range = (v..1) + super rescue range + end + jit_entry(0) + jit_entry(0) + jit_entry(0/1r) + "#), @"(0/1)..1"); + } + + // Multiple exception raises during keyword argument evaluation: each + // raise needs correct ISEQ for catch table lookup. + #[test] + fn test_iseq_on_raise_on_ensure() { + assert_snapshot!(inspect(r#" + def raise_a = raise "a" + def raise_b = raise "b" + def raise_c = raise "c" + + def foo(a: raise_a, b: raise_b, c: raise_c) + [a, b, c] + end + + def test_a + foo(b: 2, c: 3) + rescue RuntimeError => e + e.message + end + + def test_b + foo(a: 1, c: 3) + rescue RuntimeError => e + e.message + end + + def test_c + foo(a: 1, b: 2) + rescue RuntimeError => e + e.message + end + + def test + [test_a, test_b, test_c] + end + + test + test + "#), @r#"["a", "b", "c"]"#); + } + + // Send fallback (e.g. method_missing) calls into the interpreter, which + // reads cfp->iseq via GET_ISEQ(). gen_prepare_non_leaf_call writes the + // iseq to JITFrame, but GET_ISEQ reads cfp->iseq directly. This test + // ensures the interpreter can resolve the caller iseq for backtraces. + #[test] + fn test_send_fallback_caller_location() { + assert_snapshot!(inspect(r#" + def callee = caller_locations(1, 1)[0].label + def test = callee + test + test + "#), @r#""Object#test""#); + } + + // A send fallback may throw (e.g. via method_missing raising). The + // interpreter must be able to find the correct rescue handler in the + // caller's ISEQ catch table. This exercises throw through send fallback. + #[test] + fn test_send_fallback_throw() { + assert_snapshot!(inspect(r#" + class Foo + def method_missing(name, *) = raise("no #{name}") + end + def test + Foo.new.bar + rescue RuntimeError => e + e.message + end + test + test + "#), @r#""no bar""#); + } + + // Proc.new inside a block passed via invokeblock captures the caller's + // block_code. When the JIT compiles the caller, block_code must be + // correctly available for the proc to work. + #[test] + fn test_proc_from_invokeblock() { + assert_snapshot!(inspect(" + def capture_block(&blk) = blk + def test = capture_block { 42 } + test + test.call + "), @"42"); + } + + // binding() called from a JIT-compiled callee must see the correct + // source location (iseq + pc) of the caller frame. + #[test] + fn test_binding_source_location() { + assert_snapshot!(inspect(r#" + def callee = binding + def test = callee + test + b = test + b.source_location[1] > 0 + "#), @"true"); + } + + // $~ (Regexp special variable) is stored via svar which walks the EP + // chain to find the LEP. rb_vm_svar_lep uses rb_zjit_cfp_has_iseq to + // skip C frames, so it must work correctly with JITFrame. + #[test] + fn test_svar_regexp_match() { + assert_snapshot!(inspect(r#" + def test(s) + s =~ /hello/ + $~ + end + test("hello world") + test("hello world").to_s + "#), @r#""hello""#); + } + + // C function calls with rb_block_call (like Array#each, Enumerable#map) + // write an ifunc to cfp->block_code after the JIT pushes the C frame. + // GC must mark and relocate this ifunc. This test exercises the code + // path fixed by "Fix ZJIT segfault: write block_code for C frames and + // fix GC marking". + #[test] + fn test_cfunc_block_code_gc() { + assert_snapshot!(inspect(" + def test + # Use a cfunc that calls back into Ruby with a block (rb_block_call) + [1, 2, 3].map { |x| x.to_s } + end + test + test + "), @r#"["1", "2", "3"]"#); + } + + // Multiple levels of cfunc-with-block: a JIT-compiled method calls a + // cfunc that yields, and the block itself calls another cfunc that + // yields. Each C frame's block_code must be properly initialized. + #[test] + fn test_nested_cfunc_with_block() { + assert_snapshot!(inspect(" + def test + [1, 2].flat_map { |x| [x, x + 10].map { |y| y * 2 } } + end + test + test + "), @"[2, 22, 4, 24]"); + } +} diff --git a/zjit/src/lib.rs b/zjit/src/lib.rs index 50a48399c2c2d1..1440b6ff6942e6 100644 --- a/zjit/src/lib.rs +++ b/zjit/src/lib.rs @@ -29,6 +29,7 @@ mod profile; mod invariants; mod bitset; mod gc; +mod jit_frame; mod payload; mod json; mod ttycolors; diff --git a/zjit/src/payload.rs b/zjit/src/payload.rs index 660e2a80ce9101..567c6e9eea10f8 100644 --- a/zjit/src/payload.rs +++ b/zjit/src/payload.rs @@ -4,6 +4,8 @@ use crate::codegen::IseqCallRef; use crate::stats::CompileError; use crate::{cruby::*, profile::IseqProfile, virtualmem::CodePtr}; +pub use crate::jit_frame::JITFrame; + /// This is all the data ZJIT stores on an ISEQ. We mark objects in this struct on GC. #[derive(Debug)] pub struct IseqPayload { diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 6c61bc77efd314..db69bcdf655a70 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -8,6 +8,7 @@ use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; use crate::options::{get_option, rb_zjit_prepare_options}; +use crate::payload::JITFrame; use crate::stats::{Counters, InsnCounters, PerfettoTracer}; use crate::virtualmem::CodePtr; use std::sync::atomic::AtomicUsize; @@ -70,6 +71,9 @@ pub struct ZJITState { /// Perfetto tracer for --zjit-trace-exits perfetto_tracer: Option, + + /// Frame metadata for ISEQ and C calls that are known at compile time + jit_frames: Vec<*const JITFrame>, } /// Tracks the initialization progress @@ -147,6 +151,7 @@ impl ZJITState { ccall_counter_pointers: HashMap::new(), iseq_calls_count_pointers: HashMap::new(), perfetto_tracer, + jit_frames: vec![], }; unsafe { ZJIT_STATE = Enabled(zjit_state); } @@ -186,6 +191,10 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } + pub fn get_jit_frames() -> &'static mut Vec<*const JITFrame> { + &mut ZJITState::get_instance().jit_frames + } + pub fn get_method_annotations() -> &'static cruby_methods::Annotations { &ZJITState::get_instance().method_annotations } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 330b9f32bfbe2f..537813fee7438d 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -163,6 +163,7 @@ make_counters! { gc_time_ns, invalidation_time_ns, + compiled_side_exit_count, side_exit_size, compile_side_exit_time_ns, From 31dba124482faa6e95c1ce30af31bbcc784fbb49 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 15:43:28 -0700 Subject: [PATCH 11/23] ZJIT: Rename CFP_JIT_RETURN to CFP_HAS_JIT_RETURN --- cont.c | 2 +- vm.c | 4 ++-- yjit.c | 2 +- zjit.h | 10 +++++----- zjit/src/backend/lir.rs | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cont.c b/cont.c index 9370058f554798..9f6e300045b324 100644 --- a/cont.c +++ b/cont.c @@ -1480,7 +1480,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { + if (CFP_HAS_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { ((rb_control_frame_t *)cfp)->jit_return = leave_exit; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/vm.c b/vm.c index 71144b98e9b469..edb7de7d829e19 100644 --- a/vm.c +++ b/vm.c @@ -2852,7 +2852,7 @@ zjit_materialize_frames(rb_control_frame_t *cfp) if (!rb_zjit_enabled_p) return; while (true) { - if (CFP_JIT_RETURN(cfp)) { + if (CFP_HAS_JIT_RETURN(cfp)) { const zjit_jit_frame_t *jit_frame = (const zjit_jit_frame_t *)cfp->jit_return; cfp->pc = jit_frame->pc; cfp->iseq = (rb_iseq_t *)jit_frame->iseq; @@ -3671,7 +3671,7 @@ rb_execution_context_update(rb_execution_context_t *ec) while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); - if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { zjit_jit_frame_t *jit_frame = (zjit_jit_frame_t *)cfp->jit_return; if (jit_frame->iseq) { // ISEQ frame with JITFrame: relocate iseq in JITFrame diff --git a/yjit.c b/yjit.c index 46565cb6c0d035..a53e66c339be89 100644 --- a/yjit.c +++ b/yjit.c @@ -480,7 +480,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // If it's a FINISH frame, just normally exit with a non-Qundef value. cfp->jit_return = leave_exit; } - else if (CFP_JIT_RETURN(cfp)) { + else if (CFP_HAS_JIT_RETURN(cfp)) { while (!VM_FRAME_FINISHED_P(cfp)) { if (cfp->jit_return == leave_exit) { // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on diff --git a/zjit.h b/zjit.h index e0cdf985f133e0..8216f39e4ff1b5 100644 --- a/zjit.h +++ b/zjit.h @@ -63,7 +63,7 @@ enum zjit_poison_values { // YJIT also uses jit_return (as a return address), so this must only return // true when ZJIT is enabled and has set jit_return to a JITFrame pointer. static inline bool -CFP_JIT_RETURN(const rb_control_frame_t *cfp) +CFP_HAS_JIT_RETURN(const rb_control_frame_t *cfp) { if (!rb_zjit_enabled_p) return false; #if USE_ZJIT @@ -78,7 +78,7 @@ CFP_JIT_RETURN(const rb_control_frame_t *cfp) static inline bool rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) { - if (CFP_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq != NULL; + if (CFP_HAS_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq != NULL; return !!cfp->iseq; } @@ -88,14 +88,14 @@ rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) static inline bool rb_zjit_cfp_has_pc(const rb_control_frame_t *cfp) { - if (CFP_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->pc != NULL; + if (CFP_HAS_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->pc != NULL; return !!cfp->pc; } static inline const VALUE* rb_zjit_cfp_pc(const rb_control_frame_t *cfp) { - if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; } else { @@ -106,7 +106,7 @@ rb_zjit_cfp_pc(const rb_control_frame_t *cfp) static inline const rb_iseq_t* rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) { - if (rb_zjit_enabled_p && CFP_JIT_RETURN(cfp)) { + if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; } else { diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 624fcd0389f6f2..1b44234df8e249 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -2673,7 +2673,7 @@ impl Assembler // Clear jit_return to fully materialize the frame. This must happen // before any C call in the exit path (e.g. no_profile_send_recompile) // because that C call can trigger GC, which walks the stack and would - // hit the CFP_JIT_RETURN assertion if jit_return still holds the + // hit the CFP_HAS_JIT_RETURN assertion if jit_return still holds the // runtime_checks poison value (JIT_RETURN_POISON). asm_comment!(asm, "clear cfp->jit_return"); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into()); From 9c7c28ff67ae6e79cfb7628dcc44607f01a131d2 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 15:43:50 -0700 Subject: [PATCH 12/23] ZJIT: Simplify cfp accessor functions in zjit.h Remove redundant rb_zjit_enabled_p checks from rb_zjit_cfp_pc and rb_zjit_cfp_iseq since CFP_HAS_JIT_RETURN already checks it. Rewrite rb_zjit_cfp_has_iseq and rb_zjit_cfp_has_pc as thin wrappers around rb_zjit_cfp_iseq and rb_zjit_cfp_pc. --- zjit.h | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/zjit.h b/zjit.h index 8216f39e4ff1b5..47fed9b8a04e50 100644 --- a/zjit.h +++ b/zjit.h @@ -72,14 +72,31 @@ CFP_HAS_JIT_RETURN(const rb_control_frame_t *cfp) return !!cfp->jit_return; } +static inline const VALUE* +rb_zjit_cfp_pc(const rb_control_frame_t *cfp) +{ + if (CFP_HAS_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; + } + return cfp->pc; +} + +static inline const rb_iseq_t* +rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) +{ + if (CFP_HAS_JIT_RETURN(cfp)) { + return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; + } + return cfp->iseq; +} + // Returns true if cfp has an ISEQ, either directly or via JITFrame. // When JITFrame is present, it is authoritative (cfp->iseq may be stale). // C frames with JITFrame have iseq=NULL, so this returns false for them. static inline bool rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) { - if (CFP_HAS_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq != NULL; - return !!cfp->iseq; + return !!rb_zjit_cfp_iseq(cfp); } // Returns true if cfp has a PC, either directly or via JITFrame. @@ -88,30 +105,7 @@ rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) static inline bool rb_zjit_cfp_has_pc(const rb_control_frame_t *cfp) { - if (CFP_HAS_JIT_RETURN(cfp)) return ((const zjit_jit_frame_t *)cfp->jit_return)->pc != NULL; - return !!cfp->pc; -} - -static inline const VALUE* -rb_zjit_cfp_pc(const rb_control_frame_t *cfp) -{ - if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { - return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; - } - else { - return cfp->pc; - } -} - -static inline const rb_iseq_t* -rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) -{ - if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { - return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; - } - else { - return cfp->iseq; - } + return !!rb_zjit_cfp_pc(cfp); } #endif // #ifndef ZJIT_H From c5ed7416a13b58f55d7b76d8c6b5bf03d4931eee Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 15:44:36 -0700 Subject: [PATCH 13/23] ZJIT: Add doc comments to JITFrame and split into two constructors Add documentation to JITFrame fields explaining their purpose and nullability. Split JITFrame::new into new_iseq and new_cfunc constructors to make the two distinct use cases explicit. --- zjit/src/codegen.rs | 4 ++-- zjit/src/jit_frame.rs | 29 +++++++++++++++++++++++------ 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 260351b595bd8e..817a52aaa68707 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2744,7 +2744,7 @@ fn gen_save_pc_for_gc(asm: &mut Assembler, state: &FrameState) { if let Some(pc) = PC_POISON { asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_PC), Opnd::const_ptr(pc)); } - let jit_frame = JITFrame::new(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); + let jit_frame = JITFrame::new_iseq(next_pc, state.iseq, !iseq_may_write_block_code(state.iseq)); asm.mov(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } @@ -2894,7 +2894,7 @@ fn gen_push_frame(asm: &mut Assembler, argc: usize, state: &FrameState, frame: C // Without this, stale data from a previous frame occupying this CFP slot // can be used as an ifunc pointer, causing a segfault. asm.mov(cfp_opnd(RUBY_OFFSET_CFP_BLOCK_CODE), 0.into()); - let jit_frame = JITFrame::new(std::ptr::null(), std::ptr::null(), false); + let jit_frame = JITFrame::new_cfunc(); asm.mov(cfp_opnd(RUBY_OFFSET_CFP_JIT_RETURN), Opnd::const_ptr(jit_frame)); } diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs index 899d567177450a..7626100137b455 100644 --- a/zjit/src/jit_frame.rs +++ b/zjit/src/jit_frame.rs @@ -4,20 +4,37 @@ use crate::state::ZJITState; #[derive(Debug)] #[repr(C)] pub struct JITFrame { + /// Program counter for this frame, used for backtraces and GC. + /// NULL for C frames (they don't have a Ruby PC). pub pc: *const VALUE, - pub iseq: IseqPtr, // marked in rb_execution_context_mark + /// The ISEQ this frame belongs to. Marked via rb_execution_context_mark. + /// NULL for C frames. + pub iseq: IseqPtr, + /// Whether to materialize block_code when this frame is materialized. + /// True when the ISEQ doesn't contain send/invokesuper/invokeblock + /// (which write block_code themselves), so we must restore it. + /// Always false for C frames. pub materialize_block_code: bool, } impl JITFrame { - /// Allocate a new JITFrame on the heap and register it with ZJITState. - /// Returns a raw pointer that remains valid for the lifetime of the process. - pub fn new(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { - let jit_frame = Box::new(JITFrame { pc, iseq, materialize_block_code }); - let raw_ptr = Box::into_raw(jit_frame) as *const _; + /// Allocate a JITFrame on the heap, register it with ZJITState, and return + /// a raw pointer that remains valid for the lifetime of the process. + fn alloc(jit_frame: JITFrame) -> *const Self { + let raw_ptr = Box::into_raw(Box::new(jit_frame)) as *const _; ZJITState::get_jit_frames().push(raw_ptr); raw_ptr } + + /// Create a JITFrame for an ISEQ frame. + pub fn new_iseq(pc: *const VALUE, iseq: IseqPtr, materialize_block_code: bool) -> *const Self { + Self::alloc(JITFrame { pc, iseq, materialize_block_code }) + } + + /// Create a JITFrame for a C frame (no PC, no ISEQ). + pub fn new_cfunc() -> *const Self { + Self::alloc(JITFrame { pc: std::ptr::null(), iseq: std::ptr::null(), materialize_block_code: false }) + } } #[cfg(test)] From 984feb2bedf73b58b7b80974d364861c6d0f2ec5 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 15:44:53 -0700 Subject: [PATCH 14/23] ZJIT: Import JITFrame directly instead of via payload --- zjit/src/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zjit/src/state.rs b/zjit/src/state.rs index db69bcdf655a70..1f5c56883594f2 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -8,7 +8,7 @@ use std::sync::atomic::Ordering; use crate::invariants::Invariants; use crate::asm::CodeBlock; use crate::options::{get_option, rb_zjit_prepare_options}; -use crate::payload::JITFrame; +use crate::jit_frame::JITFrame; use crate::stats::{Counters, InsnCounters, PerfettoTracer}; use crate::virtualmem::CodePtr; use std::sync::atomic::AtomicUsize; From 219f7bbf2626e46c42c67da26ab2e0ed1c569868 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 15:46:10 -0700 Subject: [PATCH 15/23] ZJIT: Have JITFrame mark and update its own GC pointers Add mark() and update_references() methods to JITFrame so GC logic is encapsulated. Store *mut JITFrame in ZJITState to avoid unsafe const-to-mut casts in gc.rs. --- zjit/src/gc.rs | 14 +++----------- zjit/src/jit_frame.rs | 23 ++++++++++++++++++++--- zjit/src/state.rs | 4 ++-- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/zjit/src/gc.rs b/zjit/src/gc.rs index 1a0990bc6d5a3a..239b71d5f48754 100644 --- a/zjit/src/gc.rs +++ b/zjit/src/gc.rs @@ -3,7 +3,7 @@ use std::ptr::null; use std::{ffi::c_void, ops::Range}; use crate::{cruby::*, state::ZJITState, stats::with_time_stat, virtualmem::CodePtr}; -use crate::payload::{IseqPayload, IseqVersionRef, JITFrame, get_or_create_iseq_payload}; +use crate::payload::{IseqPayload, IseqVersionRef, get_or_create_iseq_payload}; use crate::stats::Counter::gc_time_ns; /// GC callback for marking GC objects in the per-ISEQ payload. @@ -95,13 +95,7 @@ pub extern "C" fn rb_zjit_root_update_references() { // but JITFrames not on the stack also need their iseq pointers updated // because the JIT code will reuse them on the next call. for &jit_frame in ZJITState::get_jit_frames().iter() { - let old_iseq = unsafe { (*jit_frame).iseq }; - if !old_iseq.is_null() { - let new_iseq = unsafe { rb_gc_location(VALUE::from(old_iseq)) }.as_iseq(); - if old_iseq != new_iseq { - unsafe { (*(jit_frame as *mut JITFrame)).iseq = new_iseq; } - } - } + unsafe { &mut *jit_frame }.update_references(); } } @@ -227,8 +221,6 @@ pub extern "C" fn rb_zjit_root_mark() { return; } for &jit_frame in ZJITState::get_jit_frames().iter() { - if !unsafe { (*jit_frame).iseq }.is_null() { - unsafe { rb_gc_mark_movable(VALUE::from((*jit_frame).iseq)); } - } + unsafe { &*jit_frame }.mark(); } } diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs index 7626100137b455..1ab56d8360d73d 100644 --- a/zjit/src/jit_frame.rs +++ b/zjit/src/jit_frame.rs @@ -1,4 +1,4 @@ -use crate::cruby::{IseqPtr, VALUE}; +use crate::cruby::{IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; use crate::state::ZJITState; #[derive(Debug)] @@ -21,9 +21,9 @@ impl JITFrame { /// Allocate a JITFrame on the heap, register it with ZJITState, and return /// a raw pointer that remains valid for the lifetime of the process. fn alloc(jit_frame: JITFrame) -> *const Self { - let raw_ptr = Box::into_raw(Box::new(jit_frame)) as *const _; + let raw_ptr = Box::into_raw(Box::new(jit_frame)); ZJITState::get_jit_frames().push(raw_ptr); - raw_ptr + raw_ptr as *const _ } /// Create a JITFrame for an ISEQ frame. @@ -35,6 +35,23 @@ impl JITFrame { pub fn new_cfunc() -> *const Self { Self::alloc(JITFrame { pc: std::ptr::null(), iseq: std::ptr::null(), materialize_block_code: false }) } + + /// Mark the iseq pointer for GC. Called from rb_zjit_root_mark. + pub fn mark(&self) { + if !self.iseq.is_null() { + unsafe { rb_gc_mark_movable(VALUE::from(self.iseq)); } + } + } + + /// Update the iseq pointer after GC compaction. + pub fn update_references(&mut self) { + if !self.iseq.is_null() { + let new_iseq = unsafe { rb_gc_location(VALUE::from(self.iseq)) }.as_iseq(); + if self.iseq != new_iseq { + self.iseq = new_iseq; + } + } + } } #[cfg(test)] diff --git a/zjit/src/state.rs b/zjit/src/state.rs index 1f5c56883594f2..813e23dd4140c0 100644 --- a/zjit/src/state.rs +++ b/zjit/src/state.rs @@ -73,7 +73,7 @@ pub struct ZJITState { perfetto_tracer: Option, /// Frame metadata for ISEQ and C calls that are known at compile time - jit_frames: Vec<*const JITFrame>, + jit_frames: Vec<*mut JITFrame>, } /// Tracks the initialization progress @@ -191,7 +191,7 @@ impl ZJITState { &mut ZJITState::get_instance().invariants } - pub fn get_jit_frames() -> &'static mut Vec<*const JITFrame> { + pub fn get_jit_frames() -> &'static mut Vec<*mut JITFrame> { &mut ZJITState::get_instance().jit_frames } From 59fe446e8f80c67d31c12609a8bb39dba6f48ec5 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 17:12:59 -0700 Subject: [PATCH 16/23] ZJIT: Use bare opcode variant in iseq_may_write_block_code Add rb_iseq_bare_opcode_at_pc which uses rb_vm_insn_addr2insn to return the base instruction, stripping trace/zjit variants. Use it in iseq_may_write_block_code to simplify the match arms. --- depend | 1 + jit.c | 14 ++++++++++++++ zjit/bindgen/src/main.rs | 1 + zjit/src/codegen.rs | 12 +++++------- zjit/src/cruby_bindings.inc.rs | 4 ++++ 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/depend b/depend index 6b5d7ecd40f494..eef68cd3fcb09d 100644 --- a/depend +++ b/depend @@ -7830,6 +7830,7 @@ jit.$(OBJEXT): $(top_srcdir)/internal/basic_operators.h jit.$(OBJEXT): $(top_srcdir)/internal/bits.h jit.$(OBJEXT): $(top_srcdir)/internal/box.h jit.$(OBJEXT): $(top_srcdir)/internal/class.h +jit.$(OBJEXT): $(top_srcdir)/internal/compile.h jit.$(OBJEXT): $(top_srcdir)/internal/compilers.h jit.$(OBJEXT): $(top_srcdir)/internal/fixnum.h jit.$(OBJEXT): $(top_srcdir)/internal/gc.h diff --git a/jit.c b/jit.c index 0ff573f9e116c1..c1a7453b0ddcf5 100644 --- a/jit.c +++ b/jit.c @@ -12,6 +12,7 @@ #include "insns.inc" #include "insns_info.inc" #include "iseq.h" +#include "internal/compile.h" #include "internal/gc.h" #include "vm_sync.h" #include "internal/fixnum.h" @@ -83,6 +84,19 @@ rb_iseq_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) return rb_vm_insn_addr2opcode((const void *)at_pc); } +// Get the bare opcode given a program counter. Always returns the base +// instruction, stripping trace/zjit variants. +int +rb_iseq_bare_opcode_at_pc(const rb_iseq_t *iseq, const VALUE *pc) +{ + if (OPT_DIRECT_THREADED_CODE || OPT_CALL_THREADED_CODE) { + RUBY_ASSERT_ALWAYS(FL_TEST_RAW((VALUE)iseq, ISEQ_TRANSLATED)); + } + + const VALUE at_pc = *pc; + return rb_vm_insn_addr2insn((const void *)at_pc); +} + unsigned long rb_RSTRING_LEN(VALUE str) { diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 33dd4eac1e1311..209623ffa2b653 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -276,6 +276,7 @@ fn main() { .allowlist_function("rb_iseq_(get|set)_zjit_payload") .allowlist_function("rb_iseq_pc_at_idx") .allowlist_function("rb_iseq_opcode_at_pc") + .allowlist_function("rb_iseq_bare_opcode_at_pc") .allowlist_function("rb_jit_reserve_addr_space") .allowlist_function("rb_jit_mark_writable") .allowlist_function("rb_jit_mark_executable") diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 817a52aaa68707..493033a6b1abe4 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -2704,7 +2704,7 @@ fn gen_incr_send_fallback_counter(asm: &mut Assembler, reason: SendFallbackReaso } /// Check if an ISEQ contains instructions that may write to block_code -/// (send, sendforward, invokesuper, invokesuperforward, invokeblock, and their trace variants). +/// (send, sendforward, invokesuper, invokesuperforward, invokeblock). /// These instructions call vm_caller_setup_arg_block which writes to cfp->block_code. #[allow(non_upper_case_globals)] fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { @@ -2713,14 +2713,12 @@ fn iseq_may_write_block_code(iseq: IseqPtr) -> bool { while insn_idx < encoded_size { let pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; - let opcode = unsafe { rb_iseq_opcode_at_pc(iseq, pc) } as u32; + let opcode = unsafe { rb_iseq_bare_opcode_at_pc(iseq, pc) } as u32; match opcode { - YARVINSN_send | YARVINSN_trace_send | - YARVINSN_sendforward | YARVINSN_trace_sendforward | - YARVINSN_invokesuper | YARVINSN_trace_invokesuper | - YARVINSN_invokesuperforward | YARVINSN_trace_invokesuperforward | - YARVINSN_invokeblock | YARVINSN_trace_invokeblock => { + YARVINSN_send | YARVINSN_sendforward | + YARVINSN_invokesuper | YARVINSN_invokesuperforward | + YARVINSN_invokeblock => { return true; } _ => {} diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index b270929da7c31e..9a453bdd8f257e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2181,6 +2181,10 @@ unsafe extern "C" { pub fn rb_iseq_encoded_size(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint; pub fn rb_iseq_pc_at_idx(iseq: *const rb_iseq_t, insn_idx: u32) -> *mut VALUE; pub fn rb_iseq_opcode_at_pc(iseq: *const rb_iseq_t, pc: *const VALUE) -> ::std::os::raw::c_int; + pub fn rb_iseq_bare_opcode_at_pc( + iseq: *const rb_iseq_t, + pc: *const VALUE, + ) -> ::std::os::raw::c_int; pub fn rb_RSTRING_LEN(str_: VALUE) -> ::std::os::raw::c_ulong; pub fn rb_RSTRING_PTR(str_: VALUE) -> *mut ::std::os::raw::c_char; pub fn rb_insn_name(insn: VALUE) -> *const ::std::os::raw::c_char; From 29755f4d5cc88e6cc8605e415299641caed38122 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 17:13:05 -0700 Subject: [PATCH 17/23] ZJIT: Move jit_return clearing from rb_set_cfp_pc into codegen The jit_return = 0 write in rb_set_cfp_pc was only needed by ZJIT. Move it to the codegen call site next to where iseq is also set, making the intent clearer. --- jit.c | 3 --- zjit/src/codegen.rs | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/jit.c b/jit.c index c1a7453b0ddcf5..a321a835326458 100644 --- a/jit.c +++ b/jit.c @@ -537,9 +537,6 @@ void rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) { cfp->pc = pc; - if (rb_zjit_enabled_p) { - cfp->jit_return = 0; - } } void diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 493033a6b1abe4..c59dfbe525f7e6 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -3067,11 +3067,13 @@ c_callable! { let iseq_call = unsafe { Rc::from_raw(iseq_call_ptr as *const IseqCall) }; let iseq = iseq_call.iseq.get(); let entry_insn_idxs = crate::hir::jit_entry_insns(iseq); - // gen_push_frame() doesn't set PC, so we need to set them before exit. - // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC. + // gen_push_frame() doesn't set PC or ISEQ, so we need to set them before exit. + // function_stub_hit_body() may allocate and call gc_validate_pc(), so we always set PC and ISEQ. + // Clear jit_return so the interpreter reads cfp->pc and cfp->iseq directly. let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; unsafe { (*cfp).iseq = iseq }; + unsafe { (*cfp).jit_return = std::ptr::null_mut() }; } with_vm_lock(src_loc!(), || { From bfcc31ab11686ea85b3785b5664c267f56bfac0d Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 17:35:51 -0700 Subject: [PATCH 18/23] ZJIT: Define JITFrame struct in C and import via bindgen Make zjit.h the single source of truth for the JITFrame struct layout. Remove the manual #[repr(C)] definition from jit_frame.rs and replace it with a type alias to the bindgen-generated zjit_jit_frame. --- zjit.h | 14 +++++++++++--- zjit/bindgen/src/main.rs | 3 +++ zjit/src/cruby_bindings.inc.rs | 7 +++++++ zjit/src/jit_frame.rs | 19 ++++--------------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/zjit.h b/zjit.h index 47fed9b8a04e50..a1353f4bc7dd83 100644 --- a/zjit.h +++ b/zjit.h @@ -9,11 +9,19 @@ # define ZJIT_STATS (USE_ZJIT && RUBY_DEBUG) #endif -// JITFrame is a C ABI compatible struct defined in Rust with #[repr(C)]. -// C code can read its fields directly without calling Rust accessor functions. +// JITFrame is defined here as the single source of truth and imported into +// Rust via bindgen. C code reads fields directly; Rust uses an impl block. typedef struct zjit_jit_frame { + // Program counter for this frame, used for backtraces and GC. + // NULL for C frames (they don't have a Ruby PC). const VALUE *pc; - const rb_iseq_t *iseq; // marked in rb_execution_context_mark + // The ISEQ this frame belongs to. Marked via rb_execution_context_mark. + // NULL for C frames. + const rb_iseq_t *iseq; + // Whether to materialize block_code when this frame is materialized. + // True when the ISEQ doesn't contain send/invokesuper/invokeblock + // (which write block_code themselves), so we must restore it. + // Always false for C frames. bool materialize_block_code; } zjit_jit_frame_t; diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index 209623ffa2b653..b3f390026bc144 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -69,6 +69,9 @@ fn main() { // Import YARV bytecode instruction constants .allowlist_type("ruby_vminsn_type") + // JITFrame struct defined in zjit.h, imported into Rust via bindgen + .allowlist_type("zjit_jit_frame") + .allowlist_type("ruby_special_consts") .allowlist_function("rb_utf8_str_new") .allowlist_function("rb_str_buf_append") diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 9a453bdd8f257e..9a82d6a29f6cf3 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1910,6 +1910,13 @@ pub const DEFINED_REF: defined_type = 15; pub const DEFINED_FUNC: defined_type = 16; pub const DEFINED_CONST_FROM: defined_type = 17; pub type defined_type = u32; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct zjit_jit_frame { + pub pc: *const VALUE, + pub iseq: *const rb_iseq_t, + pub materialize_block_code: bool, +} pub const ZJIT_JIT_RETURN_POISON: zjit_poison_values = 2; pub type zjit_poison_values = u32; pub const ISEQ_BODY_OFFSET_PARAM: zjit_struct_offsets = 16; diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs index 1ab56d8360d73d..c4a0f5ee717401 100644 --- a/zjit/src/jit_frame.rs +++ b/zjit/src/jit_frame.rs @@ -1,21 +1,10 @@ use crate::cruby::{IseqPtr, VALUE, rb_gc_mark_movable, rb_gc_location}; +use crate::cruby::zjit_jit_frame; use crate::state::ZJITState; -#[derive(Debug)] -#[repr(C)] -pub struct JITFrame { - /// Program counter for this frame, used for backtraces and GC. - /// NULL for C frames (they don't have a Ruby PC). - pub pc: *const VALUE, - /// The ISEQ this frame belongs to. Marked via rb_execution_context_mark. - /// NULL for C frames. - pub iseq: IseqPtr, - /// Whether to materialize block_code when this frame is materialized. - /// True when the ISEQ doesn't contain send/invokesuper/invokeblock - /// (which write block_code themselves), so we must restore it. - /// Always false for C frames. - pub materialize_block_code: bool, -} +/// JITFrame struct is defined in zjit.h (the single source of truth) and +/// imported into Rust via bindgen. See zjit.h for field documentation. +pub type JITFrame = zjit_jit_frame; impl JITFrame { /// Allocate a JITFrame on the heap, register it with ZJITState, and return From 4e5b0d3132c39b690ac50d7a49e96a95b8271067 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Mar 2026 17:41:06 -0700 Subject: [PATCH 19/23] ZJIT: Have JITFrame relocate itself via Rust function Expose rb_zjit_jit_frame_update_references so vm.c can call into Rust to relocate the iseq pointer during GC compaction, instead of manually manipulating JITFrame fields in C. This keeps JITFrame field access consolidated in Rust. --- vm.c | 6 +----- zjit.h | 2 ++ zjit/src/jit_frame.rs | 7 +++++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/vm.c b/vm.c index edb7de7d829e19..40f764d90b4774 100644 --- a/vm.c +++ b/vm.c @@ -3672,11 +3672,7 @@ rb_execution_context_update(rb_execution_context_t *ec) const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { - zjit_jit_frame_t *jit_frame = (zjit_jit_frame_t *)cfp->jit_return; - if (jit_frame->iseq) { - // ISEQ frame with JITFrame: relocate iseq in JITFrame - jit_frame->iseq = (const rb_iseq_t *)rb_gc_location((VALUE)jit_frame->iseq); - } + rb_zjit_jit_frame_update_references((zjit_jit_frame_t *)cfp->jit_return); // block_code must always be relocated. For ISEQ frames, the JIT caller // may have written it (gen_block_handler_specval) for passing blocks. // For C frames, rb_iterate0 may have written an ifunc to block_code diff --git a/zjit.h b/zjit.h index a1353f4bc7dd83..9d865b49afe1ac 100644 --- a/zjit.h +++ b/zjit.h @@ -45,6 +45,7 @@ void rb_zjit_before_ractor_spawn(void); void rb_zjit_tracing_invalidate_all(void); void rb_zjit_invalidate_no_singleton_class(VALUE klass); void rb_zjit_invalidate_root_box(void); +void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame); #else #define rb_zjit_entry 0 static inline void rb_zjit_compile_iseq(const rb_iseq_t *iseq, rb_execution_context_t *ec, bool jit_exception) {} @@ -58,6 +59,7 @@ static inline void rb_zjit_before_ractor_spawn(void) {} static inline void rb_zjit_tracing_invalidate_all(void) {} static inline void rb_zjit_invalidate_no_singleton_class(VALUE klass) {} static inline void rb_zjit_invalidate_root_box(void) {} +static inline void rb_zjit_jit_frame_update_references(zjit_jit_frame_t *jit_frame) {} #endif // #if USE_ZJIT #define rb_zjit_enabled_p (rb_zjit_entry != 0) diff --git a/zjit/src/jit_frame.rs b/zjit/src/jit_frame.rs index c4a0f5ee717401..f7133daab2ee79 100644 --- a/zjit/src/jit_frame.rs +++ b/zjit/src/jit_frame.rs @@ -43,6 +43,13 @@ impl JITFrame { } } +/// Update the iseq pointer in an on-stack JITFrame during GC compaction. +/// Called from rb_execution_context_update in vm.c. +#[unsafe(no_mangle)] +pub extern "C" fn rb_zjit_jit_frame_update_references(jit_frame: *mut JITFrame) { + unsafe { &mut *jit_frame }.update_references(); +} + #[cfg(test)] mod tests { use crate::cruby::{eval, inspect}; From 226f37059ec5f3ea3a1417e0bab630c64dbc8ac3 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 14:28:01 -0700 Subject: [PATCH 20/23] Rename cfp->iseq to cfp->_iseq and rb_zjit_cfp_iseq to rb_cfp_iseq Per tenderlove's review, every cfp->iseq read should go through an accessor that accounts for JITFrame. Rename the field to _iseq so direct access causes a compile error, and rename the accessors to drop the zjit prefix (rb_cfp_iseq, rb_cfp_has_iseq, rb_cfp_pc, rb_cfp_has_pc). --- cont.c | 8 ++--- error.c | 2 +- eval.c | 4 +-- gc.c | 10 +++--- jit.c | 2 +- thread.c | 10 +++--- vm.c | 60 +++++++++++++++---------------- vm_backtrace.c | 42 +++++++++++----------- vm_core.h | 6 ++-- vm_dump.c | 28 +++++++-------- vm_eval.c | 14 ++++---- vm_exec.h | 8 ++--- vm_insnhelper.c | 66 +++++++++++++++++----------------- vm_insnhelper.h | 2 +- vm_method.c | 4 +-- vm_trace.c | 8 ++--- zjit.h | 16 ++++----- zjit/src/codegen.rs | 2 +- zjit/src/cruby_bindings.inc.rs | 2 +- 19 files changed, 147 insertions(+), 147 deletions(-) diff --git a/cont.c b/cont.c index 9f6e300045b324..7ceaa7c7745921 100644 --- a/cont.c +++ b/cont.c @@ -1456,8 +1456,8 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (rb_zjit_cfp_has_pc(cfp) && rb_zjit_cfp_has_iseq(cfp)) { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + if (rb_cfp_has_pc(cfp) && rb_cfp_has_iseq(cfp)) { + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { callback(iseq, data); } @@ -1580,8 +1580,8 @@ show_vm_pcs(const rb_control_frame_t *cfp, int i=0; while (cfp != end_of_cfp) { int pc = 0; - if (cfp->iseq) { - pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + if (rb_cfp_iseq(cfp)) { + pc = cfp->pc - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; } fprintf(stderr, "%2d pc: %d\n", i++, pc); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/error.c b/error.c index 7f277017dd3ed9..b25f0b8e78e31c 100644 --- a/error.c +++ b/error.c @@ -2376,7 +2376,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method) rb_ivar_set(exc, id_name, method); err_init_recv(exc, recv); if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_zjit_cfp_iseq(cfp))); + rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_cfp_iseq(cfp))); } return exc; } diff --git a/eval.c b/eval.c index a30a36e4745a94..46ee992c7a9e09 100644 --- a/eval.c +++ b/eval.c @@ -2011,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec) while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { if (VM_FRAME_RUBYFRAME_P(cfp)) { - if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) { + if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } - else if (ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE && + else if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE && !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) && !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; diff --git a/gc.c b/gc.c index 59e4a3f1fe7959..ca26318d9589ed 100644 --- a/gc.c +++ b/gc.c @@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_pc(cfp)) { - const VALUE *iseq_encoded = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; - const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_size; - RUBY_ASSERT(rb_zjit_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); - RUBY_ASSERT(rb_zjit_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_cfp_has_pc(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_size; + RUBY_ASSERT(rb_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(rb_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } diff --git a/jit.c b/jit.c index a321a835326458..68dd13b73d1636 100644 --- a/jit.c +++ b/jit.c @@ -403,7 +403,7 @@ rb_get_ec_cfp(const rb_execution_context_t *ec) const rb_iseq_t * rb_get_cfp_iseq(struct rb_control_frame_struct *cfp) { - return cfp->iseq; + return rb_cfp_iseq(cfp); } VALUE * diff --git a/thread.c b/thread.c index 8586b4811a8df5..42d44dfdf1cc2f 100644 --- a/thread.c +++ b/thread.c @@ -5906,7 +5906,7 @@ static void update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(cfp->iseq); + VALUE coverage = rb_iseq_coverage(rb_cfp_iseq(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE lines = RARRAY_AREF(coverage, COVERAGE_INDEX_LINES); if (lines) { @@ -5916,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) VALUE num; void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset); if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) { - rb_iseq_clear_event_flags(cfp->iseq, rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); + rb_iseq_clear_event_flags(rb_cfp_iseq(cfp), rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); rb_ary_push(lines, LONG2FIX(line + 1)); return; } @@ -5937,12 +5937,12 @@ static void update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(cfp->iseq); + VALUE coverage = rb_iseq_coverage(rb_cfp_iseq(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES); if (branches) { - long pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(cfp->iseq)->iseq_encoded - 1; - long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(cfp->iseq), pc)), count; + long pc = rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded - 1; + long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(rb_cfp_iseq(cfp)), pc)), count; VALUE counters = RARRAY_AREF(branches, 1); VALUE num = RARRAY_AREF(counters, idx); count = FIX2LONG(num) + 1; diff --git a/vm.c b/vm.c index 40f764d90b4774..c54705efb7bdf9 100644 --- a/vm.c +++ b/vm.c @@ -516,7 +516,7 @@ rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) static inline rb_jit_func_t yjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); // Increment the ISEQ's call counter and trigger JIT compilation if not compiled @@ -536,7 +536,7 @@ yjit_compile(rb_execution_context_t *ec) static inline rb_jit_func_t zjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); if (body->jit_entry == NULL) { @@ -605,7 +605,7 @@ jit_exec(rb_execution_context_t *ec) static inline rb_jit_func_t jit_compile_exception(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = ec->cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); #if USE_ZJIT @@ -941,7 +941,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (rb_zjit_cfp_has_iseq(cfp)) { + if (rb_cfp_has_iseq(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1119,7 +1119,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } @@ -1689,8 +1689,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, rb_zjit_cfp_iseq(cfp)); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(rb_zjit_cfp_iseq(ruby_level_cfp))->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, rb_cfp_iseq(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(rb_cfp_iseq(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1997,9 +1997,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (!rb_zjit_cfp_has_pc(cfp) || !rb_zjit_cfp_has_iseq(cfp)) { + while (!rb_cfp_has_pc(cfp) || !rb_cfp_has_iseq(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_zjit_cfp_iseq(cfp); + struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_cfp_iseq(cfp); return ifunc->svar_lep; } else { @@ -2082,7 +2082,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(rb_zjit_cfp_iseq(cfp))); + return RSTRING_PTR(rb_iseq_path(rb_cfp_iseq(cfp))); } else { return 0; @@ -2111,7 +2111,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(rb_zjit_cfp_iseq(cfp)); + return rb_iseq_path(rb_cfp_iseq(cfp)); } else { if (pline) *pline = 0; @@ -2593,7 +2593,7 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(ec->cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); rb_hook_list_t *local_hooks = NULL; unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; if (RB_UNLIKELY(local_hooks_cnt > 0)) { @@ -2855,7 +2855,7 @@ zjit_materialize_frames(rb_control_frame_t *cfp) if (CFP_HAS_JIT_RETURN(cfp)) { const zjit_jit_frame_t *jit_frame = (const zjit_jit_frame_t *)cfp->jit_return; cfp->pc = jit_frame->pc; - cfp->iseq = (rb_iseq_t *)jit_frame->iseq; + cfp->_iseq = (rb_iseq_t *)jit_frame->iseq; if (jit_frame->materialize_block_code) { cfp->block_code = NULL; } @@ -2889,7 +2889,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (rb_zjit_cfp_pc(ec->cfp) == 0 || rb_zjit_cfp_iseq(ec->cfp) == 0) { + while (rb_cfp_pc(ec->cfp) == 0 || rb_cfp_iseq(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2903,7 +2903,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded; + epc = rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2916,7 +2916,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2950,7 +2950,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } if (state == TAG_RAISE) { - ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2966,7 +2966,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2982,7 +2982,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { zjit_materialize_frames(cfp); - cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; + cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -3000,7 +3000,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -3013,7 +3013,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } else if (entry->type == type) { zjit_materialize_frames(cfp); - cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + entry->cont; + cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -3027,7 +3027,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -3047,9 +3047,9 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V const int arg_size = 1; rb_iseq_check(catch_iseq); - zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->iseq + zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->_iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -3680,7 +3680,7 @@ rb_execution_context_update(rb_execution_context_t *ec) cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); } else { - cfp->iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->iseq); + cfp->_iseq = (rb_iseq_t *)rb_gc_location((VALUE)cfp->_iseq); cfp->block_code = (void *)rb_gc_location((VALUE)cfp->block_code); } @@ -3733,7 +3733,7 @@ rb_execution_context_mark(const rb_execution_context_t *ec) VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)rb_zjit_cfp_iseq(cfp)); + rb_gc_mark_movable((VALUE)rb_cfp_iseq(cfp)); // Mark block_code directly (not through rb_zjit_cfp_block_code) // because rb_iterate0 may write a valid ifunc after JIT frame push. rb_gc_mark_movable((VALUE)cfp->block_code); @@ -4575,7 +4575,7 @@ Init_VM(void) rb_root_fiber_obj_setup(th); rb_vm_register_global_object((VALUE)iseq); - th->ec->cfp->iseq = iseq; + th->ec->cfp->_iseq = iseq; th->ec->cfp->pc = ISEQ_BODY(iseq)->iseq_encoded; th->ec->cfp->self = th->top_self; @@ -4611,7 +4611,7 @@ rb_vm_set_progname(VALUE filename) --cfp; filename = rb_str_new_frozen(filename); - rb_iseq_pathobj_set(cfp->iseq, filename, rb_iseq_realpath(cfp->iseq)); + rb_iseq_pathobj_set(rb_cfp_iseq(cfp), filename, rb_iseq_realpath(rb_cfp_iseq(cfp))); } extern const struct st_hash_type rb_fstring_hash_type; @@ -4943,7 +4943,7 @@ vm_analysis_operand(int insn, int n, VALUE op) HASH_ASET(ihash, INT2FIX(n), ophash); } /* intern */ - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(rb_cfp_iseq(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); /* set count */ if (NIL_P(cv = rb_hash_aref(ophash, valstr))) { @@ -5145,7 +5145,7 @@ vm_collect_usage_operand(int insn, int n, VALUE op) if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) { VALUE valstr; - valstr = rb_insn_operand_intern(GET_EC()->cfp->iseq, insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(rb_cfp_iseq(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); RB_GC_GUARD(valstr); diff --git a/vm_backtrace.c b/vm_backtrace.c index 674cfe24274c24..022249afa029d6 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -102,9 +102,9 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && rb_zjit_cfp_has_iseq(cfp)) { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); - int line = calc_lineno(iseq, rb_zjit_cfp_pc(cfp)); + if (VM_FRAME_RUBYFRAME_P(cfp) && rb_cfp_has_iseq(cfp)) { + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + int line = calc_lineno(iseq, rb_cfp_pc(cfp)); if (line != 0) { return line; } @@ -618,7 +618,7 @@ backtrace_size(const rb_execution_context_t *ec) static bool is_rescue_or_ensure_frame(const rb_control_frame_t *cfp) { - enum rb_iseq_type type = ISEQ_BODY(rb_zjit_cfp_iseq(cfp))->type; + enum rb_iseq_type type = ISEQ_BODY(rb_cfp_iseq(cfp))->type; return type == ISEQ_TYPE_RESCUE || type == ISEQ_TYPE_ENSURE; } @@ -688,17 +688,17 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (rb_zjit_cfp_has_iseq(cfp)) { - if (rb_zjit_cfp_has_pc(cfp)) { + if (rb_cfp_has_iseq(cfp)) { + if (rb_cfp_has_pc(cfp)) { if (start_frame > 0) { start_frame--; } else { - bool internal = is_internal_location(rb_zjit_cfp_iseq(cfp)); + bool internal = is_internal_location(rb_cfp_iseq(cfp)); if (skip_internal && internal) continue; if (!skip_next_frame) { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); - const VALUE *pc = rb_zjit_cfp_pc(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const VALUE *pc = rb_cfp_pc(cfp); if (internal && backpatch_counter > 0) { // To keep only one internal frame, discard the previous backpatch frames bt->backtrace_size -= backpatch_counter; @@ -753,10 +753,10 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (rb_zjit_cfp_has_iseq(cfp) && rb_zjit_cfp_has_pc(cfp) && !(skip_internal && is_internal_location(rb_zjit_cfp_iseq(cfp)))) { + if (rb_cfp_has_iseq(cfp) && rb_cfp_has_pc(cfp) && !(skip_internal && is_internal_location(rb_cfp_iseq(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc - bt_backpatch_loc(backpatch_counter, loc, rb_zjit_cfp_iseq(cfp), rb_zjit_cfp_pc(cfp)); - RB_OBJ_WRITTEN(btobj, Qundef, rb_zjit_cfp_iseq(cfp)); + bt_backpatch_loc(backpatch_counter, loc, rb_cfp_iseq(cfp), rb_cfp_pc(cfp)); + RB_OBJ_WRITTEN(btobj, Qundef, rb_cfp_iseq(cfp)); if (do_yield) { bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); } @@ -1020,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (rb_zjit_cfp_has_iseq(cfp)) { - if (rb_zjit_cfp_has_pc(cfp)) { + if (rb_cfp_has_iseq(cfp)) { + if (rb_cfp_has_pc(cfp)) { iter_iseq(arg, cfp); } } @@ -1053,8 +1053,8 @@ oldbt_init(void *ptr, size_t dmy) static void oldbt_iter_iseq(void *ptr, const rb_control_frame_t *cfp) { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); - const VALUE *pc = rb_zjit_cfp_pc(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const VALUE *pc = rb_cfp_pc(cfp); struct oldbt_arg *arg = (struct oldbt_arg *)ptr; VALUE file = arg->filename = rb_iseq_path(iseq); VALUE name = ISEQ_BODY(iseq)->location.label; @@ -1551,7 +1551,7 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) { struct collect_caller_bindings_data *data = (struct collect_caller_bindings_data *)arg; VALUE frame = rb_ary_new2(6); - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); rb_ary_store(frame, CALLER_BINDING_SELF, cfp->self); rb_ary_store(frame, CALLER_BINDING_CLASS, get_klass(cfp)); @@ -1562,7 +1562,7 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) rb_backtrace_location_t *loc = &data->bt->backtrace[data->bt->backtrace_size++]; RB_OBJ_WRITE(data->btobj, &loc->cme, rb_vm_frame_method_entry(cfp)); RB_OBJ_WRITE(data->btobj, &loc->iseq, iseq); - loc->pc = rb_zjit_cfp_pc(cfp); + loc->pc = rb_cfp_pc(cfp); VALUE vloc = location_create(loc, (void *)data->btobj); rb_ary_store(frame, CALLER_BINDING_LOC, vloc); @@ -1747,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; i 0) { start--; continue; @@ -1755,7 +1755,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b /* record frame info */ cme = rb_vm_frame_method_entry_unchecked(cfp); - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); if (cme && cme->def->type == VM_METHOD_TYPE_ISEQ) { buff[i] = (VALUE)cme; } @@ -1764,7 +1764,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b } if (lines) { - const VALUE *pc = rb_zjit_cfp_pc(cfp); + const VALUE *pc = rb_cfp_pc(cfp); VALUE *iseq_encoded = ISEQ_BODY(iseq)->iseq_encoded; VALUE *pc_end = iseq_encoded + ISEQ_BODY(iseq)->iseq_size; diff --git a/vm_core.h b/vm_core.h index 3bf1d51b0921ce..85664e18b8396b 100644 --- a/vm_core.h +++ b/vm_core.h @@ -920,7 +920,7 @@ struct rb_block { typedef struct rb_control_frame_struct { const VALUE *pc; // cfp[0] VALUE *sp; // cfp[1] - const rb_iseq_t *iseq; // cfp[2] + const rb_iseq_t *_iseq; // cfp[2] -- use rb_cfp_iseq(cfp) to read VALUE self; // cfp[3] / block[0] const VALUE *ep; // cfp[4] / block[1] const void *block_code; // cfp[5] / block[2] -- iseq, ifunc, or forwarded block handler @@ -1532,10 +1532,10 @@ static inline int VM_FRAME_CFRAME_P(const rb_control_frame_t *cfp) { int cframe_p = VM_ENV_FLAGS(cfp->ep, VM_FRAME_FLAG_CFRAME) != 0; - // With ZJIT lightweight frames, cfp->iseq may be stale (not yet materialized), + // With ZJIT lightweight frames, cfp->_iseq may be stale (not yet materialized), // so skip this assertion when jit_return is set (zjit.h is not available here). VM_ASSERT(cfp->jit_return || - RUBY_VM_NORMAL_ISEQ_P(cfp->iseq) != cframe_p || + RUBY_VM_NORMAL_ISEQ_P(cfp->_iseq) != cframe_p || (VM_FRAME_TYPE(cfp) & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY); return cframe_p; } diff --git a/vm_dump.c b/vm_dump.c index 8e92566ce347ad..52b8fb0ad7332a 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -119,8 +119,8 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (rb_zjit_cfp_has_iseq(cfp)) { - iseq = rb_zjit_cfp_iseq(cfp); + if (rb_cfp_has_iseq(cfp)) { + iseq = rb_cfp_iseq(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) if (RUBY_VM_IFUNC_P(iseq)) { iseq_name = ""; @@ -132,8 +132,8 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c line = -1; } else { - if (rb_zjit_cfp_has_pc(cfp)) { - pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; + if (rb_cfp_has_pc(cfp)) { + pc = rb_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -210,7 +210,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c if (ISEQ_BODY(iseq)->local_table_size > 0) { kprintf(" lvars:\n"); for (unsigned int i=0; ilocal_table_size; i++) { - const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1; + const VALUE *argv = cfp->ep - ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1; kprintf(" %s: %s\n", rb_id2name(ISEQ_BODY(iseq)->local_table[i]), rb_raw_obj_info(buff, 0x100, argv[i])); @@ -341,9 +341,9 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && rb_zjit_cfp_has_iseq(cfp)) { + if (cfp && rb_cfp_has_iseq(cfp)) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - const rb_iseq_t *resolved_iseq = rb_zjit_cfp_iseq(cfp); + const rb_iseq_t *resolved_iseq = rb_cfp_iseq(cfp); if (RUBY_VM_IFUNC_P(resolved_iseq)) { iseq_name = ""; } @@ -354,9 +354,9 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro line = -1; } else { - if (rb_zjit_cfp_has_pc(cfp)) { + if (rb_cfp_has_pc(cfp)) { iseq = resolved_iseq; - pc = rb_zjit_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; + pc = rb_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -567,9 +567,9 @@ static const VALUE * vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - const VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE; + const VALUE *bp = prev_cfp->sp + ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { bp += 1; } return bp; @@ -584,7 +584,7 @@ vm_stack_dump_each(const rb_execution_context_t *ec, const rb_control_frame_t *c const VALUE *ep = cfp->ep; if (VM_FRAME_RUBYFRAME_P(cfp)) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); argc = ISEQ_BODY(iseq)->param.lead_num; local_table_size = ISEQ_BODY(iseq)->local_table_size; } @@ -655,7 +655,7 @@ rb_vmdebug_debug_print_register(const rb_execution_context_t *ec, FILE *errout) ptrdiff_t cfpi; if (VM_FRAME_RUBYFRAME_P(cfp)) { - pc = cfp->pc - ISEQ_BODY(cfp->iseq)->iseq_encoded; + pc = cfp->pc - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; } if (ep < 0 || (size_t)ep > ec->vm_stack_size) { @@ -680,7 +680,7 @@ rb_vmdebug_thread_dump_regs(VALUE thval, FILE *errout) bool rb_vmdebug_debug_print_pre(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE *_pc, FILE *errout) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); if (iseq != 0) { ptrdiff_t pc = _pc - ISEQ_BODY(iseq)->iseq_encoded; diff --git a/vm_eval.c b/vm_eval.c index 02fd83ef699923..3fd47b9dab61ac 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1988,12 +1988,12 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li block.as.captured = *VM_CFP_TO_CAPTURED_BLOCK(cfp); block.as.captured.self = self; - block.as.captured.code.iseq = rb_zjit_cfp_iseq(cfp); + block.as.captured.code.iseq = rb_cfp_iseq(cfp); block.type = block_type_iseq; // EP is not escaped to the heap here, but captured and reused by another frame. // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT. - rb_zjit_invalidate_no_ep_escape(rb_zjit_cfp_iseq(cfp)); + rb_zjit_invalidate_no_ep_escape(rb_cfp_iseq(cfp)); iseq = eval_make_iseq(src, file, line, &block); if (!iseq) { @@ -2773,9 +2773,9 @@ rb_f_local_variables(VALUE _) local_var_list_init(&vars); while (cfp) { - if (cfp->iseq) { - for (i = 0; i < ISEQ_BODY(cfp->iseq)->local_table_size; i++) { - local_var_list_add(&vars, ISEQ_BODY(cfp->iseq)->local_table[i]); + if (rb_cfp_iseq(cfp)) { + for (i = 0; i < ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size; i++) { + local_var_list_add(&vars, ISEQ_BODY(rb_cfp_iseq(cfp))->local_table[i]); } } if (!VM_ENV_LOCAL_P(cfp->ep)) { @@ -2849,7 +2849,7 @@ rb_current_realfilepath(void) rb_control_frame_t *cfp = ec->cfp; cfp = vm_get_ruby_level_caller_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)); if (cfp != NULL) { - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); VALUE path = rb_iseq_realpath(iseq); if (RTEST(path)) return path; // eval context @@ -2879,7 +2879,7 @@ struct vm_ifunc * rb_current_ifunc(void) { // Search VM_FRAME_MAGIC_IFUNC to see ifunc imemos put on the iseq field. - VALUE ifunc = (VALUE)GET_EC()->cfp->iseq; + VALUE ifunc = (VALUE)rb_cfp_iseq(GET_EC()->cfp); RUBY_ASSERT_ALWAYS(imemo_type_p(ifunc, imemo_ifunc)); return (struct vm_ifunc *)ifunc; } diff --git a/vm_exec.h b/vm_exec.h index 340fadff517223..8939eb8c0cbd9a 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -68,10 +68,10 @@ error ! #define INSN_ENTRY_SIG(insn) \ if (0) { \ ruby_debug_printf("exec: %s@(%"PRIdPTRDIFF", %"PRIdPTRDIFF")@%s:%u\n", #insn, \ - (reg_pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded), \ - (reg_cfp->pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded), \ - RSTRING_PTR(rb_iseq_path(reg_cfp->iseq)), \ - rb_iseq_line_no(reg_cfp->iseq, reg_pc - ISEQ_BODY(reg_cfp->iseq)->iseq_encoded)); \ + (reg_pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded), \ + (reg_cfp->pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded), \ + RSTRING_PTR(rb_iseq_path(rb_cfp_iseq(reg_cfp))), \ + rb_iseq_line_no(rb_cfp_iseq(reg_cfp), reg_pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded)); \ } #define INSN_DISPATCH_SIG(insn) diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 1b5a40bc9768af..3d9f6062aac4e8 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -405,7 +405,7 @@ vm_push_frame(rb_execution_context_t *ec, *cfp = (const struct rb_control_frame_struct) { .pc = pc, .sp = sp, - .iseq = iseq, + ._iseq = iseq, .self = self, .ep = sp - 1, .block_code = NULL, @@ -1767,16 +1767,16 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c escape_cfp = reg_cfp; while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) { - if (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { + if (ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp); ep = escape_cfp->ep; - base_iseq = rb_zjit_cfp_iseq(escape_cfp); + base_iseq = rb_cfp_iseq(escape_cfp); } else { ep = VM_ENV_PREV_EP(ep); base_iseq = ISEQ_BODY(base_iseq)->parent_iseq; escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep); - VM_ASSERT(rb_zjit_cfp_iseq(escape_cfp) == base_iseq); + VM_ASSERT(rb_cfp_iseq(escape_cfp) == base_iseq); } } @@ -1790,8 +1790,8 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c while (escape_cfp < eocfp) { if (escape_cfp->ep == ep) { - const rb_iseq_t *const iseq = rb_zjit_cfp_iseq(escape_cfp); - const VALUE epc = rb_zjit_cfp_pc(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; + const rb_iseq_t *const iseq = rb_cfp_iseq(escape_cfp); + const VALUE epc = rb_cfp_pc(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table; unsigned int i; @@ -1850,7 +1850,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c if (lep == target_lep && VM_FRAME_RUBYFRAME_P(escape_cfp) && - ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { + ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { in_class_frame = 1; target_lep = 0; } @@ -1880,7 +1880,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) { - switch (ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type) { + switch (ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type) { case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: if (toplevel) { @@ -1894,7 +1894,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } break; case ISEQ_TYPE_EVAL: { - const rb_iseq_t *is = rb_zjit_cfp_iseq(escape_cfp); + const rb_iseq_t *is = rb_cfp_iseq(escape_cfp); enum rb_iseq_type t = ISEQ_BODY(is)->type; while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) { if (!(is = ISEQ_BODY(is)->parent_iseq)) break; @@ -1912,7 +1912,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } - if (escape_cfp->ep == target_lep && ISEQ_BODY(rb_zjit_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_METHOD) { + if (escape_cfp->ep == target_lep && ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_METHOD) { if (target_ep == NULL) { goto valid_return; } @@ -2376,7 +2376,7 @@ vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct } #endif - return vm_search_method_slowpath0((VALUE)rb_zjit_cfp_iseq(reg_cfp), cd, klass); + return vm_search_method_slowpath0((VALUE)rb_cfp_iseq(reg_cfp), cd, klass); } static const struct rb_callable_method_entry_struct * @@ -2661,18 +2661,18 @@ vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - if (cfp->iseq && VM_FRAME_RUBYFRAME_P(cfp)) { - VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE; + if (rb_cfp_iseq(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) { + VALUE *bp = prev_cfp->sp + ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int lts = ISEQ_BODY(cfp->iseq)->local_table_size; - int params = ISEQ_BODY(cfp->iseq)->param.size; + if (ISEQ_BODY(rb_cfp_iseq(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int lts = ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size; + int params = ISEQ_BODY(rb_cfp_iseq(cfp))->param.size; CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local bp += vm_ci_argc(ci); } - if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { /* adjust `self' */ bp += 1; } @@ -3406,7 +3406,7 @@ vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct iseq = env->iseq; } else { // Otherwise use the lep to find the caller - iseq = rb_vm_search_cf_from_ep(ec, cfp, lep)->iseq; + iseq = rb_cfp_iseq(rb_vm_search_cf_from_ep(ec, cfp, lep)); } // Our local storage is below the args we need to copy @@ -4613,8 +4613,8 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) { rb_control_frame_t *top_cfp = cfp; - if (cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_BLOCK) { - const rb_iseq_t *local_iseq = ISEQ_BODY(cfp->iseq)->local_iseq; + if (rb_cfp_iseq(cfp) && ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_BLOCK) { + const rb_iseq_t *local_iseq = ISEQ_BODY(rb_cfp_iseq(cfp))->local_iseq; do { cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -4622,7 +4622,7 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) /* TODO: orphan block */ return top_cfp; } - } while (cfp->iseq != local_iseq); + } while (rb_cfp_iseq(cfp) != local_iseq); } return cfp; } @@ -4705,7 +4705,7 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c if (ref_cme) { if (calling->cd->cc) { const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); - RB_OBJ_WRITE(rb_zjit_cfp_iseq(cfp), &calling->cd->cc, cc); + RB_OBJ_WRITE(rb_cfp_iseq(cfp), &calling->cd->cc, cc); return vm_call_method(ec, cfp, calling); } else { @@ -5126,7 +5126,7 @@ static const struct rb_callcache * vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE current_defined_class; - const rb_iseq_t *iseq = rb_zjit_cfp_iseq(reg_cfp); + const rb_iseq_t *iseq = rb_cfp_iseq(reg_cfp); const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp); if (!me) { @@ -6170,7 +6170,7 @@ rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6212,7 +6212,7 @@ rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(rb_zjit_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(rb_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6620,7 +6620,7 @@ rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const vm_ic_track_const_chain(GET_CFP(), ic, segments); // Undo the PC increment to get the address to this instruction // INSN_ATTR(width) == 2 - vm_ic_update(rb_zjit_cfp_iseq(GET_CFP()), ic, val, GET_EP(), rb_zjit_cfp_pc(GET_CFP()) - 2); + vm_ic_update(rb_cfp_iseq(GET_CFP()), ic, val, GET_EP(), rb_cfp_pc(GET_CFP()) - 2); } return val; } @@ -6645,7 +6645,7 @@ vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is) RB_OBJ_SET_SHAREABLE(val); } - RB_OBJ_WRITE(ec->cfp->iseq, &is->once.value, val); + RB_OBJ_WRITE(rb_cfp_iseq(ec->cfp), &is->once.value, val); /* is->once.running_thread is cleared by vm_once_clear() */ is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */ @@ -6714,7 +6714,7 @@ vm_stack_consistency_error(const rb_execution_context_t *ec, #if defined RUBY_DEVEL VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp); rb_str_cat_cstr(mesg, "\n"); - rb_str_append(mesg, rb_iseq_disasm(cfp->iseq)); + rb_str_append(mesg, rb_iseq_disasm(rb_cfp_iseq(cfp))); rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg)); #else rb_bug(stack_consistency_error, nsp, nbp); @@ -7244,7 +7244,7 @@ static VALUE rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp) { VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp)); - VM_ASSERT(ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE); + VM_ASSERT(ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE); return cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } @@ -7260,7 +7260,7 @@ vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp) return; } else { - const rb_iseq_t *iseq = reg_cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(reg_cfp); size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded; rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos); unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; @@ -7546,7 +7546,7 @@ lookup_builtin_invoker(int argc) static inline VALUE invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv) { - const bool canary_p = ISEQ_BODY(reg_cfp->iseq)->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` + const bool canary_p = ISEQ_BODY(rb_cfp_iseq(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` SETUP_CANARY(canary_p); rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr; VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr); @@ -7566,7 +7566,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, if (0) { // debug print fputs("vm_invoke_builtin_delegate: passing -> ", stderr); for (int i=0; iargc; i++) { - ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(cfp->iseq)->local_table[i+start_index])); + ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(rb_cfp_iseq(cfp))->local_table[i+start_index])); } ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc, (void *)(uintptr_t)bf->func_ptr); @@ -7576,7 +7576,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, return invoke_bf(ec, cfp, bf, NULL); } else { - const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; + const VALUE *argv = cfp->ep - ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; return invoke_bf(ec, cfp, bf, argv); } } diff --git a/vm_insnhelper.h b/vm_insnhelper.h index 6d32a7535b3308..d1bd57753d8f18 100644 --- a/vm_insnhelper.h +++ b/vm_insnhelper.h @@ -114,7 +114,7 @@ enum vm_regan_acttype { /* set current stack value as x */ /* instruction sequence C struct */ -#define GET_ISEQ() (GET_CFP()->iseq) +#define GET_ISEQ() (rb_cfp_iseq(GET_CFP())) /**********************************************************/ /* deal with variables */ diff --git a/vm_method.c b/vm_method.c index 521595d0b19666..eb0eb005eb9aac 100644 --- a/vm_method.c +++ b/vm_method.c @@ -1084,7 +1084,7 @@ rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *de cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp && (line = rb_vm_get_sourceline(cfp))) { - VALUE location = rb_ary_new3(2, rb_iseq_path(cfp->iseq), INT2FIX(line)); + VALUE location = rb_ary_new3(2, rb_iseq_path(rb_cfp_iseq(cfp)), INT2FIX(line)); rb_ary_freeze(location); RB_OBJ_SET_SHAREABLE(location); RB_OBJ_WRITE(me, &def->body.attr.location, location); @@ -2330,7 +2330,7 @@ scope_visibility_check(void) { /* Check for public/protected/private/module_function called inside a method */ rb_control_frame_t *cfp = GET_EC()->cfp+1; - if (cfp && cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD) { + if (cfp && rb_cfp_iseq(cfp) && ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD) { rb_warn("calling %s without arguments inside a method may not have the intended effect", rb_id2name(rb_frame_this_func())); } diff --git a/vm_trace.c b/vm_trace.c index 1dc176081440b0..62e03442e2d33d 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -812,7 +812,7 @@ get_path_and_lineno(const rb_execution_context_t *ec, const rb_control_frame_t * cfp = rb_vm_get_ruby_level_next_cfp(ec, cfp); if (cfp) { - const rb_iseq_t *iseq = cfp->iseq; + const rb_iseq_t *iseq = rb_cfp_iseq(cfp); *pathp = rb_iseq_path(iseq); if (event & (RUBY_EVENT_CLASS | @@ -862,7 +862,7 @@ call_trace_func(rb_event_flag_t event, VALUE proc, VALUE self, ID id, VALUE klas if (self && (filename != Qnil) && event != RUBY_EVENT_C_CALL && event != RUBY_EVENT_C_RETURN && - (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)ec->cfp->iseq, imemo_iseq))) { + (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)rb_cfp_iseq(ec->cfp), imemo_iseq))) { argv[4] = rb_binding_new(); } argv[5] = klass ? klass : Qnil; @@ -1041,7 +1041,7 @@ rb_tracearg_parameters(rb_trace_arg_t *trace_arg) if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_BLOCK && !VM_FRAME_LAMBDA_P(cfp)) { is_proc = 1; } - return rb_iseq_parameters(cfp->iseq, is_proc); + return rb_iseq_parameters(rb_cfp_iseq(cfp), is_proc); } break; } @@ -1103,7 +1103,7 @@ rb_tracearg_binding(rb_trace_arg_t *trace_arg) } cfp = rb_vm_get_binding_creatable_next_cfp(trace_arg->ec, trace_arg->cfp); - if (cfp && imemo_type_p((VALUE)cfp->iseq, imemo_iseq)) { + if (cfp && imemo_type_p((VALUE)rb_cfp_iseq(cfp), imemo_iseq)) { return rb_vm_make_binding(trace_arg->ec, cfp); } else { diff --git a/zjit.h b/zjit.h index 9d865b49afe1ac..634b7523c5b85a 100644 --- a/zjit.h +++ b/zjit.h @@ -83,7 +83,7 @@ CFP_HAS_JIT_RETURN(const rb_control_frame_t *cfp) } static inline const VALUE* -rb_zjit_cfp_pc(const rb_control_frame_t *cfp) +rb_cfp_pc(const rb_control_frame_t *cfp) { if (CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; @@ -92,30 +92,30 @@ rb_zjit_cfp_pc(const rb_control_frame_t *cfp) } static inline const rb_iseq_t* -rb_zjit_cfp_iseq(const rb_control_frame_t *cfp) +rb_cfp_iseq(const rb_control_frame_t *cfp) { if (CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; } - return cfp->iseq; + return cfp->_iseq; } // Returns true if cfp has an ISEQ, either directly or via JITFrame. -// When JITFrame is present, it is authoritative (cfp->iseq may be stale). +// When JITFrame is present, it is authoritative (cfp->_iseq may be stale). // C frames with JITFrame have iseq=NULL, so this returns false for them. static inline bool -rb_zjit_cfp_has_iseq(const rb_control_frame_t *cfp) +rb_cfp_has_iseq(const rb_control_frame_t *cfp) { - return !!rb_zjit_cfp_iseq(cfp); + return !!rb_cfp_iseq(cfp); } // Returns true if cfp has a PC, either directly or via JITFrame. // When JITFrame is present, it is authoritative (cfp->pc may be stale/poisoned). // C frames with JITFrame have pc=NULL, so this returns false for them. static inline bool -rb_zjit_cfp_has_pc(const rb_control_frame_t *cfp) +rb_cfp_has_pc(const rb_control_frame_t *cfp) { - return !!rb_zjit_cfp_pc(cfp); + return !!rb_cfp_pc(cfp); } #endif // #ifndef ZJIT_H diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index c59dfbe525f7e6..9846a0e70a23f1 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -3072,7 +3072,7 @@ c_callable! { // Clear jit_return so the interpreter reads cfp->pc and cfp->iseq directly. let pc = unsafe { rb_iseq_pc_at_idx(iseq, entry_insn_idxs[iseq_call.jit_entry_idx.to_usize()]) }; unsafe { rb_set_cfp_pc(cfp, pc) }; - unsafe { (*cfp).iseq = iseq }; + unsafe { (*cfp)._iseq = iseq }; unsafe { (*cfp).jit_return = std::ptr::null_mut() }; } diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 9a82d6a29f6cf3..d8fdcbb84538c3 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1303,7 +1303,7 @@ pub struct rb_block__bindgen_ty_1 { pub struct rb_control_frame_struct { pub pc: *const VALUE, pub sp: *mut VALUE, - pub iseq: *const rb_iseq_t, + pub _iseq: *const rb_iseq_t, pub self_: VALUE, pub ep: *const VALUE, pub block_code: *const ::std::os::raw::c_void, From 4391c5f087086115264af287e32f14ec8911e113 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 14:51:06 -0700 Subject: [PATCH 21/23] Use cfp->_iseq directly in GET_ISEQ() macro The interpreter always has a valid _iseq field (it's written on exit from JIT code), so GET_ISEQ() doesn't need to go through rb_cfp_iseq(). Code in vm_insnhelper.c that may be called as a ZJIT fallback should use rb_cfp_iseq() instead. --- vm_insnhelper.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vm_insnhelper.h b/vm_insnhelper.h index d1bd57753d8f18..88c387ee152afa 100644 --- a/vm_insnhelper.h +++ b/vm_insnhelper.h @@ -113,8 +113,11 @@ enum vm_regan_acttype { #define SET_SV(x) (*GET_SP() = rb_ractor_confirm_belonging(x)) /* set current stack value as x */ -/* instruction sequence C struct */ -#define GET_ISEQ() (rb_cfp_iseq(GET_CFP())) +// instruction sequence C struct +// Uses cfp->_iseq directly because the interpreter always has a valid _iseq +// field (it's written on exit from JIT code). Code in vm_insnhelper.c that +// may be called as a ZJIT fallback should use rb_cfp_iseq() instead. +#define GET_ISEQ() (GET_CFP()->_iseq) /**********************************************************/ /* deal with variables */ From ee60b99b2f76a7c8810675a3634eae1935cc8261 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 15:04:29 -0700 Subject: [PATCH 22/23] Rename rb_cfp_iseq to CFP_ISEQ and rb_cfp_pc to CFP_PC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename for consistency with ISEQ_BODY naming convention. Remove the separate rb_cfp_has_iseq/rb_cfp_has_pc wrappers — CFP_ISEQ and CFP_PC return NULL for missing values, which works directly in if conditions. --- cont.c | 8 +++---- error.c | 2 +- eval.c | 4 ++-- gc.c | 10 ++++---- jit.c | 2 +- thread.c | 10 ++++---- vm.c | 52 ++++++++++++++++++++-------------------- vm_backtrace.c | 42 ++++++++++++++++---------------- vm_dump.c | 28 +++++++++++----------- vm_eval.c | 14 +++++------ vm_exec.h | 8 +++---- vm_insnhelper.c | 64 ++++++++++++++++++++++++------------------------- vm_method.c | 4 ++-- vm_trace.c | 8 +++---- zjit.h | 22 ++--------------- 15 files changed, 130 insertions(+), 148 deletions(-) diff --git a/cont.c b/cont.c index 7ceaa7c7745921..5dfc590e4513e4 100644 --- a/cont.c +++ b/cont.c @@ -1456,8 +1456,8 @@ rb_jit_cont_each_iseq(rb_iseq_callback callback, void *data) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (rb_cfp_has_pc(cfp) && rb_cfp_has_iseq(cfp)) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + if (CFP_PC(cfp) && CFP_ISEQ(cfp)) { + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (iseq && imemo_type((VALUE)iseq) == imemo_iseq) { callback(iseq, data); } @@ -1580,8 +1580,8 @@ show_vm_pcs(const rb_control_frame_t *cfp, int i=0; while (cfp != end_of_cfp) { int pc = 0; - if (rb_cfp_iseq(cfp)) { - pc = cfp->pc - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; + if (CFP_ISEQ(cfp)) { + pc = cfp->pc - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; } fprintf(stderr, "%2d pc: %d\n", i++, pc); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/error.c b/error.c index b25f0b8e78e31c..52bd3629bf2d13 100644 --- a/error.c +++ b/error.c @@ -2376,7 +2376,7 @@ name_err_init_attr(VALUE exc, VALUE recv, VALUE method) rb_ivar_set(exc, id_name, method); err_init_recv(exc, recv); if (cfp && VM_FRAME_TYPE(cfp) != VM_FRAME_MAGIC_DUMMY) { - rb_ivar_set(exc, id_iseq, rb_iseqw_new(rb_cfp_iseq(cfp))); + rb_ivar_set(exc, id_iseq, rb_iseqw_new(CFP_ISEQ(cfp))); } return exc; } diff --git a/eval.c b/eval.c index 46ee992c7a9e09..e3630c246ae1f0 100644 --- a/eval.c +++ b/eval.c @@ -2011,10 +2011,10 @@ errinfo_place(const rb_execution_context_t *ec) while (RUBY_VM_VALID_CONTROL_FRAME_P(cfp, end_cfp)) { if (VM_FRAME_RUBYFRAME_P(cfp)) { - if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } - else if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_ENSURE && + else if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_ENSURE && !THROW_DATA_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR]) && !FIXNUM_P(cfp->ep[VM_ENV_INDEX_LAST_LVAR])) { return &cfp->ep[VM_ENV_INDEX_LAST_LVAR]; diff --git a/gc.c b/gc.c index ca26318d9589ed..d6d517d6a44c9e 100644 --- a/gc.c +++ b/gc.c @@ -1000,11 +1000,11 @@ gc_validate_pc(VALUE obj) rb_execution_context_t *ec = GET_EC(); const rb_control_frame_t *cfp = ec->cfp; - if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && rb_cfp_has_pc(cfp)) { - const VALUE *iseq_encoded = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; - const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_size; - RUBY_ASSERT(rb_cfp_pc(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); - RUBY_ASSERT(rb_cfp_pc(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); + if (cfp && VM_FRAME_RUBYFRAME_P(cfp) && CFP_PC(cfp)) { + const VALUE *iseq_encoded = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; + const VALUE *iseq_encoded_end = iseq_encoded + ISEQ_BODY(CFP_ISEQ(cfp))->iseq_size; + RUBY_ASSERT(CFP_PC(cfp) >= iseq_encoded, "PC not set when allocating, breaking tracing"); + RUBY_ASSERT(CFP_PC(cfp) <= iseq_encoded_end, "PC not set when allocating, breaking tracing"); } #endif } diff --git a/jit.c b/jit.c index 68dd13b73d1636..10c845491b2271 100644 --- a/jit.c +++ b/jit.c @@ -403,7 +403,7 @@ rb_get_ec_cfp(const rb_execution_context_t *ec) const rb_iseq_t * rb_get_cfp_iseq(struct rb_control_frame_struct *cfp) { - return rb_cfp_iseq(cfp); + return CFP_ISEQ(cfp); } VALUE * diff --git a/thread.c b/thread.c index 42d44dfdf1cc2f..f876b4bd05c80e 100644 --- a/thread.c +++ b/thread.c @@ -5906,7 +5906,7 @@ static void update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(rb_cfp_iseq(cfp)); + VALUE coverage = rb_iseq_coverage(CFP_ISEQ(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE lines = RARRAY_AREF(coverage, COVERAGE_INDEX_LINES); if (lines) { @@ -5916,7 +5916,7 @@ update_line_coverage(VALUE data, const rb_trace_arg_t *trace_arg) VALUE num; void rb_iseq_clear_event_flags(const rb_iseq_t *iseq, size_t pos, rb_event_flag_t reset); if (GET_VM()->coverage_mode & COVERAGE_TARGET_ONESHOT_LINES) { - rb_iseq_clear_event_flags(rb_cfp_iseq(cfp), rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); + rb_iseq_clear_event_flags(CFP_ISEQ(cfp), CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded - 1, RUBY_EVENT_COVERAGE_LINE); rb_ary_push(lines, LONG2FIX(line + 1)); return; } @@ -5937,12 +5937,12 @@ static void update_branch_coverage(VALUE data, const rb_trace_arg_t *trace_arg) { const rb_control_frame_t *cfp = GET_EC()->cfp; - VALUE coverage = rb_iseq_coverage(rb_cfp_iseq(cfp)); + VALUE coverage = rb_iseq_coverage(CFP_ISEQ(cfp)); if (RB_TYPE_P(coverage, T_ARRAY) && !RBASIC_CLASS(coverage)) { VALUE branches = RARRAY_AREF(coverage, COVERAGE_INDEX_BRANCHES); if (branches) { - long pc = rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded - 1; - long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(rb_cfp_iseq(cfp)), pc)), count; + long pc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded - 1; + long idx = FIX2INT(RARRAY_AREF(ISEQ_PC2BRANCHINDEX(CFP_ISEQ(cfp)), pc)), count; VALUE counters = RARRAY_AREF(branches, 1); VALUE num = RARRAY_AREF(counters, idx); count = FIX2LONG(num) + 1; diff --git a/vm.c b/vm.c index c54705efb7bdf9..9282956c482b61 100644 --- a/vm.c +++ b/vm.c @@ -516,7 +516,7 @@ rb_yjit_threshold_hit(const rb_iseq_t *iseq, uint64_t entry_calls) static inline rb_jit_func_t yjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); // Increment the ISEQ's call counter and trigger JIT compilation if not compiled @@ -536,7 +536,7 @@ yjit_compile(rb_execution_context_t *ec) static inline rb_jit_func_t zjit_compile(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); if (body->jit_entry == NULL) { @@ -605,7 +605,7 @@ jit_exec(rb_execution_context_t *ec) static inline rb_jit_func_t jit_compile_exception(rb_execution_context_t *ec) { - const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); struct rb_iseq_constant_body *body = ISEQ_BODY(iseq); #if USE_ZJIT @@ -941,7 +941,7 @@ rb_control_frame_t * rb_vm_get_binding_creatable_next_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) { - if (rb_cfp_has_iseq(cfp)) { + if (CFP_ISEQ(cfp)) { return (rb_control_frame_t *)cfp; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -1119,7 +1119,7 @@ vm_make_env_each(const rb_execution_context_t * const ec, rb_control_frame_t *co } } - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (!VM_FRAME_RUBYFRAME_P(cfp)) { local_size = VM_ENV_DATA_SIZE; } @@ -1689,8 +1689,8 @@ rb_vm_make_binding(const rb_execution_context_t *ec, const rb_control_frame_t *s GetBindingPtr(bindval, bind); vm_bind_update_env(bindval, bind, envval); RB_OBJ_WRITE(bindval, &bind->block.as.captured.self, cfp->self); - RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, rb_cfp_iseq(cfp)); - RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(rb_cfp_iseq(ruby_level_cfp))->location.pathobj); + RB_OBJ_WRITE(bindval, &bind->block.as.captured.code.iseq, CFP_ISEQ(cfp)); + RB_OBJ_WRITE(bindval, &bind->pathobj, ISEQ_BODY(CFP_ISEQ(ruby_level_cfp))->location.pathobj); bind->first_lineno = rb_vm_get_sourceline(ruby_level_cfp); return bindval; @@ -1997,9 +1997,9 @@ rb_vm_invoke_proc_with_self(rb_execution_context_t *ec, rb_proc_t *proc, VALUE s VALUE * rb_vm_svar_lep(const rb_execution_context_t *ec, const rb_control_frame_t *cfp) { - while (!rb_cfp_has_pc(cfp) || !rb_cfp_has_iseq(cfp)) { + while (!CFP_PC(cfp) || !CFP_ISEQ(cfp)) { if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_IFUNC) { - struct vm_ifunc *ifunc = (struct vm_ifunc *)rb_cfp_iseq(cfp); + struct vm_ifunc *ifunc = (struct vm_ifunc *)CFP_ISEQ(cfp); return ifunc->svar_lep; } else { @@ -2082,7 +2082,7 @@ rb_sourcefile(void) const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp) { - return RSTRING_PTR(rb_iseq_path(rb_cfp_iseq(cfp))); + return RSTRING_PTR(rb_iseq_path(CFP_ISEQ(cfp))); } else { return 0; @@ -2111,7 +2111,7 @@ rb_source_location(int *pline) if (cfp && VM_FRAME_RUBYFRAME_P(cfp)) { if (pline) *pline = rb_vm_get_sourceline(cfp); - return rb_iseq_path(rb_cfp_iseq(cfp)); + return rb_iseq_path(CFP_ISEQ(cfp)); } else { if (pline) *pline = 0; @@ -2593,7 +2593,7 @@ hook_before_rewind(rb_execution_context_t *ec, bool cfp_returning_with_value, in return; } else { - const rb_iseq_t *iseq = rb_cfp_iseq(ec->cfp); + const rb_iseq_t *iseq = CFP_ISEQ(ec->cfp); rb_hook_list_t *local_hooks = NULL; unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; if (RB_UNLIKELY(local_hooks_cnt > 0)) { @@ -2889,7 +2889,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V cont_pc = cont_sp = 0; catch_iseq = NULL; - while (rb_cfp_pc(ec->cfp) == 0 || rb_cfp_iseq(ec->cfp) == 0) { + while (CFP_PC(ec->cfp) == 0 || CFP_ISEQ(ec->cfp) == 0) { if (UNLIKELY(VM_FRAME_TYPE(ec->cfp) == VM_FRAME_MAGIC_CFUNC)) { EXEC_EVENT_HOOK_AND_POP_FRAME(ec, RUBY_EVENT_C_RETURN, ec->cfp->self, rb_vm_frame_method_entry(ec->cfp)->def->original_id, @@ -2903,7 +2903,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } rb_control_frame_t *const cfp = ec->cfp; - epc = rb_cfp_pc(cfp) - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; + epc = CFP_PC(cfp) - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; escape_cfp = NULL; if (state == TAG_BREAK || state == TAG_RETURN) { @@ -2916,7 +2916,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V THROW_DATA_STATE_SET(err, state = TAG_BREAK); } else { - ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2950,7 +2950,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } if (state == TAG_RAISE) { - ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2966,7 +2966,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else if (state == TAG_RETRY) { - ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -2982,7 +2982,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V escape_cfp = THROW_DATA_CATCH_FRAME(err); if (cfp == escape_cfp) { zjit_materialize_frames(cfp); - cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + entry->cont; + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; ec->errinfo = Qnil; return Qundef; } @@ -3000,7 +3000,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V /* otherwise = dontcare */ }[state]; - ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); @@ -3013,7 +3013,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } else if (entry->type == type) { zjit_materialize_frames(cfp); - cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + entry->cont; + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + entry->cont; cfp->sp = vm_base_ptr(cfp) + entry->sp; if (state != TAG_REDO) { @@ -3027,7 +3027,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V } } else { - ct = ISEQ_BODY(rb_cfp_iseq(cfp))->catch_table; + ct = ISEQ_BODY(CFP_ISEQ(cfp))->catch_table; if (ct) for (i = 0; i < ct->size; i++) { entry = UNALIGNED_MEMBER_PTR(ct, entries[i]); if (entry->start < epc && entry->end >= epc) { @@ -3049,7 +3049,7 @@ vm_exec_handle_exception(rb_execution_context_t *ec, enum ruby_tag_type state, V rb_iseq_check(catch_iseq); zjit_materialize_frames(cfp); // vm_base_ptr looks at cfp->_iseq cfp->sp = vm_base_ptr(cfp) + cont_sp; - cfp->pc = ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded + cont_pc; + cfp->pc = ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded + cont_pc; /* push block frame */ cfp->sp[0] = (VALUE)err; @@ -3733,7 +3733,7 @@ rb_execution_context_mark(const rb_execution_context_t *ec) VM_ASSERT(!!VM_ENV_FLAGS(ep, VM_ENV_FLAG_ESCAPED) == vm_ep_in_heap_p_(ec, ep)); rb_gc_mark_movable(cfp->self); - rb_gc_mark_movable((VALUE)rb_cfp_iseq(cfp)); + rb_gc_mark_movable((VALUE)CFP_ISEQ(cfp)); // Mark block_code directly (not through rb_zjit_cfp_block_code) // because rb_iterate0 may write a valid ifunc after JIT frame push. rb_gc_mark_movable((VALUE)cfp->block_code); @@ -4611,7 +4611,7 @@ rb_vm_set_progname(VALUE filename) --cfp; filename = rb_str_new_frozen(filename); - rb_iseq_pathobj_set(rb_cfp_iseq(cfp), filename, rb_iseq_realpath(rb_cfp_iseq(cfp))); + rb_iseq_pathobj_set(CFP_ISEQ(cfp), filename, rb_iseq_realpath(CFP_ISEQ(cfp))); } extern const struct st_hash_type rb_fstring_hash_type; @@ -4943,7 +4943,7 @@ vm_analysis_operand(int insn, int n, VALUE op) HASH_ASET(ihash, INT2FIX(n), ophash); } /* intern */ - valstr = rb_insn_operand_intern(rb_cfp_iseq(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); /* set count */ if (NIL_P(cv = rb_hash_aref(ophash, valstr))) { @@ -5145,7 +5145,7 @@ vm_collect_usage_operand(int insn, int n, VALUE op) if (RUBY_DTRACE_INSN_OPERAND_ENABLED()) { VALUE valstr; - valstr = rb_insn_operand_intern(rb_cfp_iseq(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); + valstr = rb_insn_operand_intern(CFP_ISEQ(GET_EC()->cfp), insn, n, op, 0, 0, 0, 0); RUBY_DTRACE_INSN_OPERAND(RSTRING_PTR(valstr), rb_insns_name(insn)); RB_GC_GUARD(valstr); diff --git a/vm_backtrace.c b/vm_backtrace.c index 022249afa029d6..c0bc46b8caf5c7 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -102,9 +102,9 @@ calc_node_id(const rb_iseq_t *iseq, const VALUE *pc) int rb_vm_get_sourceline(const rb_control_frame_t *cfp) { - if (VM_FRAME_RUBYFRAME_P(cfp) && rb_cfp_has_iseq(cfp)) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); - int line = calc_lineno(iseq, rb_cfp_pc(cfp)); + if (VM_FRAME_RUBYFRAME_P(cfp) && CFP_ISEQ(cfp)) { + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + int line = calc_lineno(iseq, CFP_PC(cfp)); if (line != 0) { return line; } @@ -618,7 +618,7 @@ backtrace_size(const rb_execution_context_t *ec) static bool is_rescue_or_ensure_frame(const rb_control_frame_t *cfp) { - enum rb_iseq_type type = ISEQ_BODY(rb_cfp_iseq(cfp))->type; + enum rb_iseq_type type = ISEQ_BODY(CFP_ISEQ(cfp))->type; return type == ISEQ_TYPE_RESCUE || type == ISEQ_TYPE_ENSURE; } @@ -688,17 +688,17 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram } for (; cfp != end_cfp && (bt->backtrace_size < num_frames); cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (rb_cfp_has_iseq(cfp)) { - if (rb_cfp_has_pc(cfp)) { + if (CFP_ISEQ(cfp)) { + if (CFP_PC(cfp)) { if (start_frame > 0) { start_frame--; } else { - bool internal = is_internal_location(rb_cfp_iseq(cfp)); + bool internal = is_internal_location(CFP_ISEQ(cfp)); if (skip_internal && internal) continue; if (!skip_next_frame) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); - const VALUE *pc = rb_cfp_pc(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + const VALUE *pc = CFP_PC(cfp); if (internal && backpatch_counter > 0) { // To keep only one internal frame, discard the previous backpatch frames bt->backtrace_size -= backpatch_counter; @@ -753,10 +753,10 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram // is the one of the caller Ruby frame, so if the last entry is a C frame we find the caller Ruby frame here. if (backpatch_counter > 0) { for (; cfp != end_cfp; cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)) { - if (rb_cfp_has_iseq(cfp) && rb_cfp_has_pc(cfp) && !(skip_internal && is_internal_location(rb_cfp_iseq(cfp)))) { + if (CFP_ISEQ(cfp) && CFP_PC(cfp) && !(skip_internal && is_internal_location(CFP_ISEQ(cfp)))) { VM_ASSERT(!skip_next_frame); // ISEQ_TYPE_RESCUE/ISEQ_TYPE_ENSURE should have a caller Ruby ISEQ, not a cfunc - bt_backpatch_loc(backpatch_counter, loc, rb_cfp_iseq(cfp), rb_cfp_pc(cfp)); - RB_OBJ_WRITTEN(btobj, Qundef, rb_cfp_iseq(cfp)); + bt_backpatch_loc(backpatch_counter, loc, CFP_ISEQ(cfp), CFP_PC(cfp)); + RB_OBJ_WRITTEN(btobj, Qundef, CFP_ISEQ(cfp)); if (do_yield) { bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); } @@ -1020,8 +1020,8 @@ backtrace_each(const rb_execution_context_t *ec, /* SDR(); */ for (i=0, cfp = start_cfp; ivm_stack + ec->vm_stack_size) - cfp); */ - if (rb_cfp_has_iseq(cfp)) { - if (rb_cfp_has_pc(cfp)) { + if (CFP_ISEQ(cfp)) { + if (CFP_PC(cfp)) { iter_iseq(arg, cfp); } } @@ -1053,8 +1053,8 @@ oldbt_init(void *ptr, size_t dmy) static void oldbt_iter_iseq(void *ptr, const rb_control_frame_t *cfp) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); - const VALUE *pc = rb_cfp_pc(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); + const VALUE *pc = CFP_PC(cfp); struct oldbt_arg *arg = (struct oldbt_arg *)ptr; VALUE file = arg->filename = rb_iseq_path(iseq); VALUE name = ISEQ_BODY(iseq)->location.label; @@ -1551,7 +1551,7 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) { struct collect_caller_bindings_data *data = (struct collect_caller_bindings_data *)arg; VALUE frame = rb_ary_new2(6); - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); rb_ary_store(frame, CALLER_BINDING_SELF, cfp->self); rb_ary_store(frame, CALLER_BINDING_CLASS, get_klass(cfp)); @@ -1562,7 +1562,7 @@ collect_caller_bindings_iseq(void *arg, const rb_control_frame_t *cfp) rb_backtrace_location_t *loc = &data->bt->backtrace[data->bt->backtrace_size++]; RB_OBJ_WRITE(data->btobj, &loc->cme, rb_vm_frame_method_entry(cfp)); RB_OBJ_WRITE(data->btobj, &loc->iseq, iseq); - loc->pc = rb_cfp_pc(cfp); + loc->pc = CFP_PC(cfp); VALUE vloc = location_create(loc, (void *)data->btobj); rb_ary_store(frame, CALLER_BINDING_LOC, vloc); @@ -1747,7 +1747,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b end_cfp = RUBY_VM_NEXT_CONTROL_FRAME(end_cfp); for (i=0; i 0) { start--; continue; @@ -1755,7 +1755,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b /* record frame info */ cme = rb_vm_frame_method_entry_unchecked(cfp); - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (cme && cme->def->type == VM_METHOD_TYPE_ISEQ) { buff[i] = (VALUE)cme; } @@ -1764,7 +1764,7 @@ thread_profile_frames(rb_execution_context_t *ec, int start, int limit, VALUE *b } if (lines) { - const VALUE *pc = rb_cfp_pc(cfp); + const VALUE *pc = CFP_PC(cfp); VALUE *iseq_encoded = ISEQ_BODY(iseq)->iseq_encoded; VALUE *pc_end = iseq_encoded + ISEQ_BODY(iseq)->iseq_size; diff --git a/vm_dump.c b/vm_dump.c index 52b8fb0ad7332a..4a7b0df5c6c0eb 100644 --- a/vm_dump.c +++ b/vm_dump.c @@ -119,8 +119,8 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c selfstr = ""; } - if (rb_cfp_has_iseq(cfp)) { - iseq = rb_cfp_iseq(cfp); + if (CFP_ISEQ(cfp)) { + iseq = CFP_ISEQ(cfp); #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) if (RUBY_VM_IFUNC_P(iseq)) { iseq_name = ""; @@ -132,8 +132,8 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c line = -1; } else { - if (rb_cfp_has_pc(cfp)) { - pc = rb_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; + if (CFP_PC(cfp)) { + pc = CFP_PC(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -210,7 +210,7 @@ control_frame_dump(const rb_execution_context_t *ec, const rb_control_frame_t *c if (ISEQ_BODY(iseq)->local_table_size > 0) { kprintf(" lvars:\n"); for (unsigned int i=0; ilocal_table_size; i++) { - const VALUE *argv = cfp->ep - ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1; + const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1; kprintf(" %s: %s\n", rb_id2name(ISEQ_BODY(iseq)->local_table[i]), rb_raw_obj_info(buff, 0x100, argv[i])); @@ -341,9 +341,9 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro break; } - if (cfp && rb_cfp_has_iseq(cfp)) { + if (cfp && CFP_ISEQ(cfp)) { #define RUBY_VM_IFUNC_P(ptr) IMEMO_TYPE_P(ptr, imemo_ifunc) - const rb_iseq_t *resolved_iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *resolved_iseq = CFP_ISEQ(cfp); if (RUBY_VM_IFUNC_P(resolved_iseq)) { iseq_name = ""; } @@ -354,9 +354,9 @@ box_env_dump(const rb_execution_context_t *ec, const VALUE *env, const rb_contro line = -1; } else { - if (rb_cfp_has_pc(cfp)) { + if (CFP_PC(cfp)) { iseq = resolved_iseq; - pc = rb_cfp_pc(cfp) - ISEQ_BODY(iseq)->iseq_encoded; + pc = CFP_PC(cfp) - ISEQ_BODY(iseq)->iseq_encoded; iseq_name = RSTRING_PTR(ISEQ_BODY(iseq)->location.label); if (pc >= 0 && (size_t)pc <= ISEQ_BODY(iseq)->iseq_size) { line = rb_vm_get_sourceline(cfp); @@ -567,9 +567,9 @@ static const VALUE * vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - const VALUE *bp = prev_cfp->sp + ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size + VM_ENV_DATA_SIZE; + const VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { bp += 1; } return bp; @@ -584,7 +584,7 @@ vm_stack_dump_each(const rb_execution_context_t *ec, const rb_control_frame_t *c const VALUE *ep = cfp->ep; if (VM_FRAME_RUBYFRAME_P(cfp)) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); argc = ISEQ_BODY(iseq)->param.lead_num; local_table_size = ISEQ_BODY(iseq)->local_table_size; } @@ -655,7 +655,7 @@ rb_vmdebug_debug_print_register(const rb_execution_context_t *ec, FILE *errout) ptrdiff_t cfpi; if (VM_FRAME_RUBYFRAME_P(cfp)) { - pc = cfp->pc - ISEQ_BODY(rb_cfp_iseq(cfp))->iseq_encoded; + pc = cfp->pc - ISEQ_BODY(CFP_ISEQ(cfp))->iseq_encoded; } if (ep < 0 || (size_t)ep > ec->vm_stack_size) { @@ -680,7 +680,7 @@ rb_vmdebug_thread_dump_regs(VALUE thval, FILE *errout) bool rb_vmdebug_debug_print_pre(const rb_execution_context_t *ec, const rb_control_frame_t *cfp, const VALUE *_pc, FILE *errout) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); if (iseq != 0) { ptrdiff_t pc = _pc - ISEQ_BODY(iseq)->iseq_encoded; diff --git a/vm_eval.c b/vm_eval.c index 3fd47b9dab61ac..c6d5a1dd44d1ea 100644 --- a/vm_eval.c +++ b/vm_eval.c @@ -1988,12 +1988,12 @@ eval_string_with_cref(VALUE self, VALUE src, rb_cref_t *cref, VALUE file, int li block.as.captured = *VM_CFP_TO_CAPTURED_BLOCK(cfp); block.as.captured.self = self; - block.as.captured.code.iseq = rb_cfp_iseq(cfp); + block.as.captured.code.iseq = CFP_ISEQ(cfp); block.type = block_type_iseq; // EP is not escaped to the heap here, but captured and reused by another frame. // ZJIT's locals are incompatible with it unlike YJIT's, so invalidate the ISEQ for ZJIT. - rb_zjit_invalidate_no_ep_escape(rb_cfp_iseq(cfp)); + rb_zjit_invalidate_no_ep_escape(CFP_ISEQ(cfp)); iseq = eval_make_iseq(src, file, line, &block); if (!iseq) { @@ -2773,9 +2773,9 @@ rb_f_local_variables(VALUE _) local_var_list_init(&vars); while (cfp) { - if (rb_cfp_iseq(cfp)) { - for (i = 0; i < ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size; i++) { - local_var_list_add(&vars, ISEQ_BODY(rb_cfp_iseq(cfp))->local_table[i]); + if (CFP_ISEQ(cfp)) { + for (i = 0; i < ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size; i++) { + local_var_list_add(&vars, ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i]); } } if (!VM_ENV_LOCAL_P(cfp->ep)) { @@ -2849,7 +2849,7 @@ rb_current_realfilepath(void) rb_control_frame_t *cfp = ec->cfp; cfp = vm_get_ruby_level_caller_cfp(ec, RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp)); if (cfp != NULL) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); VALUE path = rb_iseq_realpath(iseq); if (RTEST(path)) return path; // eval context @@ -2879,7 +2879,7 @@ struct vm_ifunc * rb_current_ifunc(void) { // Search VM_FRAME_MAGIC_IFUNC to see ifunc imemos put on the iseq field. - VALUE ifunc = (VALUE)rb_cfp_iseq(GET_EC()->cfp); + VALUE ifunc = (VALUE)CFP_ISEQ(GET_EC()->cfp); RUBY_ASSERT_ALWAYS(imemo_type_p(ifunc, imemo_ifunc)); return (struct vm_ifunc *)ifunc; } diff --git a/vm_exec.h b/vm_exec.h index 8939eb8c0cbd9a..f0796e93ab9c18 100644 --- a/vm_exec.h +++ b/vm_exec.h @@ -68,10 +68,10 @@ error ! #define INSN_ENTRY_SIG(insn) \ if (0) { \ ruby_debug_printf("exec: %s@(%"PRIdPTRDIFF", %"PRIdPTRDIFF")@%s:%u\n", #insn, \ - (reg_pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded), \ - (reg_cfp->pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded), \ - RSTRING_PTR(rb_iseq_path(rb_cfp_iseq(reg_cfp))), \ - rb_iseq_line_no(rb_cfp_iseq(reg_cfp), reg_pc - ISEQ_BODY(rb_cfp_iseq(reg_cfp))->iseq_encoded)); \ + (reg_pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded), \ + (reg_cfp->pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded), \ + RSTRING_PTR(rb_iseq_path(CFP_ISEQ(reg_cfp))), \ + rb_iseq_line_no(CFP_ISEQ(reg_cfp), reg_pc - ISEQ_BODY(CFP_ISEQ(reg_cfp))->iseq_encoded)); \ } #define INSN_DISPATCH_SIG(insn) diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 3d9f6062aac4e8..2c9bb027528b13 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -1767,16 +1767,16 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c escape_cfp = reg_cfp; while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) { - if (ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { + if (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) { escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp); ep = escape_cfp->ep; - base_iseq = rb_cfp_iseq(escape_cfp); + base_iseq = CFP_ISEQ(escape_cfp); } else { ep = VM_ENV_PREV_EP(ep); base_iseq = ISEQ_BODY(base_iseq)->parent_iseq; escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep); - VM_ASSERT(rb_cfp_iseq(escape_cfp) == base_iseq); + VM_ASSERT(CFP_ISEQ(escape_cfp) == base_iseq); } } @@ -1790,8 +1790,8 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c while (escape_cfp < eocfp) { if (escape_cfp->ep == ep) { - const rb_iseq_t *const iseq = rb_cfp_iseq(escape_cfp); - const VALUE epc = rb_cfp_pc(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; + const rb_iseq_t *const iseq = CFP_ISEQ(escape_cfp); + const VALUE epc = CFP_PC(escape_cfp) - ISEQ_BODY(iseq)->iseq_encoded; const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table; unsigned int i; @@ -1850,7 +1850,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c if (lep == target_lep && VM_FRAME_RUBYFRAME_P(escape_cfp) && - ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_CLASS) { + ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_CLASS) { in_class_frame = 1; target_lep = 0; } @@ -1880,7 +1880,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) { - switch (ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type) { + switch (ISEQ_BODY(CFP_ISEQ(escape_cfp))->type) { case ISEQ_TYPE_TOP: case ISEQ_TYPE_MAIN: if (toplevel) { @@ -1894,7 +1894,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } break; case ISEQ_TYPE_EVAL: { - const rb_iseq_t *is = rb_cfp_iseq(escape_cfp); + const rb_iseq_t *is = CFP_ISEQ(escape_cfp); enum rb_iseq_type t = ISEQ_BODY(is)->type; while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) { if (!(is = ISEQ_BODY(is)->parent_iseq)) break; @@ -1912,7 +1912,7 @@ vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_c } } - if (escape_cfp->ep == target_lep && ISEQ_BODY(rb_cfp_iseq(escape_cfp))->type == ISEQ_TYPE_METHOD) { + if (escape_cfp->ep == target_lep && ISEQ_BODY(CFP_ISEQ(escape_cfp))->type == ISEQ_TYPE_METHOD) { if (target_ep == NULL) { goto valid_return; } @@ -2376,7 +2376,7 @@ vm_search_method_fastpath(const struct rb_control_frame_struct *reg_cfp, struct } #endif - return vm_search_method_slowpath0((VALUE)rb_cfp_iseq(reg_cfp), cd, klass); + return vm_search_method_slowpath0((VALUE)CFP_ISEQ(reg_cfp), cd, klass); } static const struct rb_callable_method_entry_struct * @@ -2661,18 +2661,18 @@ vm_base_ptr(const rb_control_frame_t *cfp) { const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); - if (rb_cfp_iseq(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) { - VALUE *bp = prev_cfp->sp + ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size + VM_ENV_DATA_SIZE; + if (CFP_ISEQ(cfp) && VM_FRAME_RUBYFRAME_P(cfp)) { + VALUE *bp = prev_cfp->sp + ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size + VM_ENV_DATA_SIZE; - if (ISEQ_BODY(rb_cfp_iseq(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { - int lts = ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size; - int params = ISEQ_BODY(rb_cfp_iseq(cfp))->param.size; + if (ISEQ_BODY(CFP_ISEQ(cfp))->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) { + int lts = ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size; + int params = ISEQ_BODY(CFP_ISEQ(cfp))->param.size; CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local bp += vm_ci_argc(ci); } - if (ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { + if (ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) { /* adjust `self' */ bp += 1; } @@ -3406,7 +3406,7 @@ vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct iseq = env->iseq; } else { // Otherwise use the lep to find the caller - iseq = rb_cfp_iseq(rb_vm_search_cf_from_ep(ec, cfp, lep)); + iseq = CFP_ISEQ(rb_vm_search_cf_from_ep(ec, cfp, lep)); } // Our local storage is below the args we need to copy @@ -4613,8 +4613,8 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) { rb_control_frame_t *top_cfp = cfp; - if (rb_cfp_iseq(cfp) && ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_BLOCK) { - const rb_iseq_t *local_iseq = ISEQ_BODY(rb_cfp_iseq(cfp))->local_iseq; + if (CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_BLOCK) { + const rb_iseq_t *local_iseq = ISEQ_BODY(CFP_ISEQ(cfp))->local_iseq; do { cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); @@ -4622,7 +4622,7 @@ current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp) /* TODO: orphan block */ return top_cfp; } - } while (rb_cfp_iseq(cfp) != local_iseq); + } while (CFP_ISEQ(cfp) != local_iseq); } return cfp; } @@ -4705,7 +4705,7 @@ vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_c if (ref_cme) { if (calling->cd->cc) { const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); - RB_OBJ_WRITE(rb_cfp_iseq(cfp), &calling->cd->cc, cc); + RB_OBJ_WRITE(CFP_ISEQ(cfp), &calling->cd->cc, cc); return vm_call_method(ec, cfp, calling); } else { @@ -5126,7 +5126,7 @@ static const struct rb_callcache * vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv) { VALUE current_defined_class; - const rb_iseq_t *iseq = rb_cfp_iseq(reg_cfp); + const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp); const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp); if (!me) { @@ -6170,7 +6170,7 @@ rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_ VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(rb_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6212,7 +6212,7 @@ rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super); if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) { - RB_OBJ_WRITE(rb_cfp_iseq(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); + RB_OBJ_WRITE(CFP_ISEQ(GET_CFP()), &cd->cc, adjusted_cd.cd.cc); } VM_EXEC(ec, val); @@ -6620,7 +6620,7 @@ rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const vm_ic_track_const_chain(GET_CFP(), ic, segments); // Undo the PC increment to get the address to this instruction // INSN_ATTR(width) == 2 - vm_ic_update(rb_cfp_iseq(GET_CFP()), ic, val, GET_EP(), rb_cfp_pc(GET_CFP()) - 2); + vm_ic_update(CFP_ISEQ(GET_CFP()), ic, val, GET_EP(), CFP_PC(GET_CFP()) - 2); } return val; } @@ -6645,7 +6645,7 @@ vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is) RB_OBJ_SET_SHAREABLE(val); } - RB_OBJ_WRITE(rb_cfp_iseq(ec->cfp), &is->once.value, val); + RB_OBJ_WRITE(CFP_ISEQ(ec->cfp), &is->once.value, val); /* is->once.running_thread is cleared by vm_once_clear() */ is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */ @@ -6714,7 +6714,7 @@ vm_stack_consistency_error(const rb_execution_context_t *ec, #if defined RUBY_DEVEL VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp); rb_str_cat_cstr(mesg, "\n"); - rb_str_append(mesg, rb_iseq_disasm(rb_cfp_iseq(cfp))); + rb_str_append(mesg, rb_iseq_disasm(CFP_ISEQ(cfp))); rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg)); #else rb_bug(stack_consistency_error, nsp, nbp); @@ -7244,7 +7244,7 @@ static VALUE rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp) { VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp)); - VM_ASSERT(ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_RESCUE); + VM_ASSERT(ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_RESCUE); return cfp->ep[VM_ENV_INDEX_LAST_LVAR]; } @@ -7260,7 +7260,7 @@ vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp) return; } else { - const rb_iseq_t *iseq = rb_cfp_iseq(reg_cfp); + const rb_iseq_t *iseq = CFP_ISEQ(reg_cfp); size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded; rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos); unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt; @@ -7546,7 +7546,7 @@ lookup_builtin_invoker(int argc) static inline VALUE invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv) { - const bool canary_p = ISEQ_BODY(rb_cfp_iseq(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` + const bool canary_p = ISEQ_BODY(CFP_ISEQ(reg_cfp))->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf` SETUP_CANARY(canary_p); rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr; VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr); @@ -7566,7 +7566,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, if (0) { // debug print fputs("vm_invoke_builtin_delegate: passing -> ", stderr); for (int i=0; iargc; i++) { - ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(rb_cfp_iseq(cfp))->local_table[i+start_index])); + ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(CFP_ISEQ(cfp))->local_table[i+start_index])); } ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc, (void *)(uintptr_t)bf->func_ptr); @@ -7576,7 +7576,7 @@ vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, return invoke_bf(ec, cfp, bf, NULL); } else { - const VALUE *argv = cfp->ep - ISEQ_BODY(rb_cfp_iseq(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; + const VALUE *argv = cfp->ep - ISEQ_BODY(CFP_ISEQ(cfp))->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index; return invoke_bf(ec, cfp, bf, argv); } } diff --git a/vm_method.c b/vm_method.c index eb0eb005eb9aac..021b06bf00109b 100644 --- a/vm_method.c +++ b/vm_method.c @@ -1084,7 +1084,7 @@ rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *de cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp); if (cfp && (line = rb_vm_get_sourceline(cfp))) { - VALUE location = rb_ary_new3(2, rb_iseq_path(rb_cfp_iseq(cfp)), INT2FIX(line)); + VALUE location = rb_ary_new3(2, rb_iseq_path(CFP_ISEQ(cfp)), INT2FIX(line)); rb_ary_freeze(location); RB_OBJ_SET_SHAREABLE(location); RB_OBJ_WRITE(me, &def->body.attr.location, location); @@ -2330,7 +2330,7 @@ scope_visibility_check(void) { /* Check for public/protected/private/module_function called inside a method */ rb_control_frame_t *cfp = GET_EC()->cfp+1; - if (cfp && rb_cfp_iseq(cfp) && ISEQ_BODY(rb_cfp_iseq(cfp))->type == ISEQ_TYPE_METHOD) { + if (cfp && CFP_ISEQ(cfp) && ISEQ_BODY(CFP_ISEQ(cfp))->type == ISEQ_TYPE_METHOD) { rb_warn("calling %s without arguments inside a method may not have the intended effect", rb_id2name(rb_frame_this_func())); } diff --git a/vm_trace.c b/vm_trace.c index 62e03442e2d33d..74bc1fce807b83 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -812,7 +812,7 @@ get_path_and_lineno(const rb_execution_context_t *ec, const rb_control_frame_t * cfp = rb_vm_get_ruby_level_next_cfp(ec, cfp); if (cfp) { - const rb_iseq_t *iseq = rb_cfp_iseq(cfp); + const rb_iseq_t *iseq = CFP_ISEQ(cfp); *pathp = rb_iseq_path(iseq); if (event & (RUBY_EVENT_CLASS | @@ -862,7 +862,7 @@ call_trace_func(rb_event_flag_t event, VALUE proc, VALUE self, ID id, VALUE klas if (self && (filename != Qnil) && event != RUBY_EVENT_C_CALL && event != RUBY_EVENT_C_RETURN && - (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)rb_cfp_iseq(ec->cfp), imemo_iseq))) { + (VM_FRAME_RUBYFRAME_P(ec->cfp) && imemo_type_p((VALUE)CFP_ISEQ(ec->cfp), imemo_iseq))) { argv[4] = rb_binding_new(); } argv[5] = klass ? klass : Qnil; @@ -1041,7 +1041,7 @@ rb_tracearg_parameters(rb_trace_arg_t *trace_arg) if (VM_FRAME_TYPE(cfp) == VM_FRAME_MAGIC_BLOCK && !VM_FRAME_LAMBDA_P(cfp)) { is_proc = 1; } - return rb_iseq_parameters(rb_cfp_iseq(cfp), is_proc); + return rb_iseq_parameters(CFP_ISEQ(cfp), is_proc); } break; } @@ -1103,7 +1103,7 @@ rb_tracearg_binding(rb_trace_arg_t *trace_arg) } cfp = rb_vm_get_binding_creatable_next_cfp(trace_arg->ec, trace_arg->cfp); - if (cfp && imemo_type_p((VALUE)rb_cfp_iseq(cfp), imemo_iseq)) { + if (cfp && imemo_type_p((VALUE)CFP_ISEQ(cfp), imemo_iseq)) { return rb_vm_make_binding(trace_arg->ec, cfp); } else { diff --git a/zjit.h b/zjit.h index 634b7523c5b85a..6981c3957a49e4 100644 --- a/zjit.h +++ b/zjit.h @@ -83,7 +83,7 @@ CFP_HAS_JIT_RETURN(const rb_control_frame_t *cfp) } static inline const VALUE* -rb_cfp_pc(const rb_control_frame_t *cfp) +CFP_PC(const rb_control_frame_t *cfp) { if (CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; @@ -92,7 +92,7 @@ rb_cfp_pc(const rb_control_frame_t *cfp) } static inline const rb_iseq_t* -rb_cfp_iseq(const rb_control_frame_t *cfp) +CFP_ISEQ(const rb_control_frame_t *cfp) { if (CFP_HAS_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; @@ -100,22 +100,4 @@ rb_cfp_iseq(const rb_control_frame_t *cfp) return cfp->_iseq; } -// Returns true if cfp has an ISEQ, either directly or via JITFrame. -// When JITFrame is present, it is authoritative (cfp->_iseq may be stale). -// C frames with JITFrame have iseq=NULL, so this returns false for them. -static inline bool -rb_cfp_has_iseq(const rb_control_frame_t *cfp) -{ - return !!rb_cfp_iseq(cfp); -} - -// Returns true if cfp has a PC, either directly or via JITFrame. -// When JITFrame is present, it is authoritative (cfp->pc may be stale/poisoned). -// C frames with JITFrame have pc=NULL, so this returns false for them. -static inline bool -rb_cfp_has_pc(const rb_control_frame_t *cfp) -{ - return !!rb_cfp_pc(cfp); -} - #endif // #ifndef ZJIT_H From 634707a7255f132eb486eaf57473925c288ef7bd Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 27 Mar 2026 15:07:59 -0700 Subject: [PATCH 23/23] Rename CFP_HAS_JIT_RETURN to CFP_JIT_RETURN Return the pointer directly instead of a bool, for consistency with CFP_ISEQ and CFP_PC. The pointer is truthy in if conditions so all callers work unchanged. --- cont.c | 2 +- vm.c | 4 ++-- yjit.c | 2 +- zjit.h | 16 ++++++++-------- zjit/src/backend/lir.rs | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cont.c b/cont.c index 5dfc590e4513e4..e5239635081629 100644 --- a/cont.c +++ b/cont.c @@ -1480,7 +1480,7 @@ rb_yjit_cancel_jit_return(void *leave_exit, void *leave_exception) const rb_control_frame_t *cfp = cont->ec->cfp; while (!RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(cont->ec, cfp)) { - if (CFP_HAS_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { + if (CFP_JIT_RETURN(cfp) && cfp->jit_return != leave_exception) { ((rb_control_frame_t *)cfp)->jit_return = leave_exit; } cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp); diff --git a/vm.c b/vm.c index 9282956c482b61..0398b9f74c9683 100644 --- a/vm.c +++ b/vm.c @@ -2852,7 +2852,7 @@ zjit_materialize_frames(rb_control_frame_t *cfp) if (!rb_zjit_enabled_p) return; while (true) { - if (CFP_HAS_JIT_RETURN(cfp)) { + if (CFP_JIT_RETURN(cfp)) { const zjit_jit_frame_t *jit_frame = (const zjit_jit_frame_t *)cfp->jit_return; cfp->pc = jit_frame->pc; cfp->_iseq = (rb_iseq_t *)jit_frame->iseq; @@ -3671,7 +3671,7 @@ rb_execution_context_update(rb_execution_context_t *ec) while (cfp != limit_cfp) { const VALUE *ep = cfp->ep; cfp->self = rb_gc_location(cfp->self); - if (rb_zjit_enabled_p && CFP_HAS_JIT_RETURN(cfp)) { + if (CFP_JIT_RETURN(cfp)) { rb_zjit_jit_frame_update_references((zjit_jit_frame_t *)cfp->jit_return); // block_code must always be relocated. For ISEQ frames, the JIT caller // may have written it (gen_block_handler_specval) for passing blocks. diff --git a/yjit.c b/yjit.c index a53e66c339be89..46565cb6c0d035 100644 --- a/yjit.c +++ b/yjit.c @@ -480,7 +480,7 @@ rb_yjit_set_exception_return(rb_control_frame_t *cfp, void *leave_exit, void *le // If it's a FINISH frame, just normally exit with a non-Qundef value. cfp->jit_return = leave_exit; } - else if (CFP_HAS_JIT_RETURN(cfp)) { + else if (CFP_JIT_RETURN(cfp)) { while (!VM_FRAME_FINISHED_P(cfp)) { if (cfp->jit_return == leave_exit) { // Unlike jit_exec(), leave_exit is not safe on a non-FINISH frame on diff --git a/zjit.h b/zjit.h index 6981c3957a49e4..e96caa257c6b91 100644 --- a/zjit.h +++ b/zjit.h @@ -69,23 +69,23 @@ enum zjit_poison_values { ZJIT_JIT_RETURN_POISON = 2, }; -// Check if cfp->jit_return holds a ZJIT lightweight frame (JITFrame pointer). +// Return the JITFrame pointer from cfp->jit_return, or NULL if not present. // YJIT also uses jit_return (as a return address), so this must only return -// true when ZJIT is enabled and has set jit_return to a JITFrame pointer. -static inline bool -CFP_HAS_JIT_RETURN(const rb_control_frame_t *cfp) +// non-NULL when ZJIT is enabled and has set jit_return to a JITFrame pointer. +static inline void * +CFP_JIT_RETURN(const rb_control_frame_t *cfp) { - if (!rb_zjit_enabled_p) return false; + if (!rb_zjit_enabled_p) return NULL; #if USE_ZJIT RUBY_ASSERT_ALWAYS(cfp->jit_return != (void *)ZJIT_JIT_RETURN_POISON); #endif - return !!cfp->jit_return; + return cfp->jit_return; } static inline const VALUE* CFP_PC(const rb_control_frame_t *cfp) { - if (CFP_HAS_JIT_RETURN(cfp)) { + if (CFP_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->pc; } return cfp->pc; @@ -94,7 +94,7 @@ CFP_PC(const rb_control_frame_t *cfp) static inline const rb_iseq_t* CFP_ISEQ(const rb_control_frame_t *cfp) { - if (CFP_HAS_JIT_RETURN(cfp)) { + if (CFP_JIT_RETURN(cfp)) { return ((const zjit_jit_frame_t *)cfp->jit_return)->iseq; } return cfp->_iseq; diff --git a/zjit/src/backend/lir.rs b/zjit/src/backend/lir.rs index 1b44234df8e249..624fcd0389f6f2 100644 --- a/zjit/src/backend/lir.rs +++ b/zjit/src/backend/lir.rs @@ -2673,7 +2673,7 @@ impl Assembler // Clear jit_return to fully materialize the frame. This must happen // before any C call in the exit path (e.g. no_profile_send_recompile) // because that C call can trigger GC, which walks the stack and would - // hit the CFP_HAS_JIT_RETURN assertion if jit_return still holds the + // hit the CFP_JIT_RETURN assertion if jit_return still holds the // runtime_checks poison value (JIT_RETURN_POISON). asm_comment!(asm, "clear cfp->jit_return"); asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into());