diff --git a/cont.c b/cont.c index 2baf52a61a9999..1382b22fce5077 100644 --- a/cont.c +++ b/cont.c @@ -475,6 +475,7 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) void * base = VirtualAlloc(0, (*count)*stride, MEM_COMMIT, PAGE_READWRITE); if (!base) { + errno = rb_w32_map_errno(GetLastError()); *count = (*count) >> 1; } else { @@ -506,93 +507,105 @@ fiber_pool_allocate_memory(size_t * count, size_t stride) } // Given an existing fiber pool, expand it by the specified number of stacks. +// // @param count the maximum number of stacks to allocate. -// @return the allocated fiber pool. +// @return the new allocation on success, or NULL on failure with errno set. +// @raise NoMemoryError if the struct or memory allocation fails. +// +// Call from fiber_pool_stack_acquire_expand with VM lock held, or from +// fiber_pool_initialize before the pool is shared across threads. // @sa fiber_pool_allocation_free static struct fiber_pool_allocation * fiber_pool_expand(struct fiber_pool * fiber_pool, size_t count) { - struct fiber_pool_allocation * allocation; - RB_VM_LOCK_ENTER(); - { - STACK_GROW_DIR_DETECTION; + STACK_GROW_DIR_DETECTION; - size_t size = fiber_pool->size; - size_t stride = size + RB_PAGE_SIZE; + size_t size = fiber_pool->size; + size_t stride = size + RB_PAGE_SIZE; - // Allocate the memory required for the stacks: - void * base = fiber_pool_allocate_memory(&count, stride); + // Allocate metadata before mmap: ruby_xmalloc (RB_ALLOC) raises on failure and + // must not run after base is mapped, or the region would leak. + struct fiber_pool_allocation * allocation = RB_ALLOC(struct fiber_pool_allocation); - if (base == NULL) { - rb_raise(rb_eFiberError, "can't alloc machine stack to fiber (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, size, ERRNOMSG); - } + // Allocate the memory required for the stacks: + void * base = fiber_pool_allocate_memory(&count, stride); - struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies; - allocation = RB_ALLOC(struct fiber_pool_allocation); + if (base == NULL) { + if (!errno) errno = ENOMEM; + ruby_xfree(allocation); + return NULL; + } + + struct fiber_pool_vacancy * vacancies = fiber_pool->vacancies; - // Initialize fiber pool allocation: - allocation->base = base; - allocation->size = size; - allocation->stride = stride; - allocation->count = count; + // Initialize fiber pool allocation: + allocation->base = base; + allocation->size = size; + allocation->stride = stride; + allocation->count = count; #ifdef FIBER_POOL_ALLOCATION_FREE - allocation->used = 0; + allocation->used = 0; #endif - allocation->pool = fiber_pool; + allocation->pool = fiber_pool; - if (DEBUG) { - fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n", - count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size); - } + if (DEBUG) { + fprintf(stderr, "fiber_pool_expand(%"PRIuSIZE"): %p, %"PRIuSIZE"/%"PRIuSIZE" x [%"PRIuSIZE":%"PRIuSIZE"]\n", + count, (void*)fiber_pool, fiber_pool->used, fiber_pool->count, size, fiber_pool->vm_stack_size); + } - // Iterate over all stacks, initializing the vacancy list: - for (size_t i = 0; i < count; i += 1) { - void * base = (char*)allocation->base + (stride * i); - void * page = (char*)base + STACK_DIR_UPPER(size, 0); + // Iterate over all stacks, initializing the vacancy list: + for (size_t i = 0; i < count; i += 1) { + void * base = (char*)allocation->base + (stride * i); + void * page = (char*)base + STACK_DIR_UPPER(size, 0); #if defined(_WIN32) - DWORD old_protect; - - if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) { - VirtualFree(allocation->base, 0, MEM_RELEASE); - rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG); - } + DWORD old_protect; + + if (!VirtualProtect(page, RB_PAGE_SIZE, PAGE_READWRITE | PAGE_GUARD, &old_protect)) { + int error = rb_w32_map_errno(GetLastError()); + VirtualFree(allocation->base, 0, MEM_RELEASE); + ruby_xfree(allocation); + errno = error; + return NULL; + } #elif defined(__wasi__) - // wasi-libc's mprotect emulation doesn't support PROT_NONE. - (void)page; + // wasi-libc's mprotect emulation doesn't support PROT_NONE. + (void)page; #else - if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) { - munmap(allocation->base, count*stride); - rb_raise(rb_eFiberError, "can't set a guard page: %s", ERRNOMSG); - } + if (mprotect(page, RB_PAGE_SIZE, PROT_NONE) < 0) { + int error = errno; + if (!error) error = ENOMEM; + munmap(allocation->base, count*stride); + ruby_xfree(allocation); + errno = error; + return NULL; + } #endif - vacancies = fiber_pool_vacancy_initialize( - fiber_pool, vacancies, - (char*)base + STACK_DIR_UPPER(0, RB_PAGE_SIZE), - size - ); + vacancies = fiber_pool_vacancy_initialize( + fiber_pool, vacancies, + (char*)base + STACK_DIR_UPPER(0, RB_PAGE_SIZE), + size + ); #ifdef FIBER_POOL_ALLOCATION_FREE - vacancies->stack.allocation = allocation; + vacancies->stack.allocation = allocation; #endif - } + } - // Insert the allocation into the head of the pool: - allocation->next = fiber_pool->allocations; + // Insert the allocation into the head of the pool: + allocation->next = fiber_pool->allocations; #ifdef FIBER_POOL_ALLOCATION_FREE - if (allocation->next) { - allocation->next->previous = allocation; - } + if (allocation->next) { + allocation->next->previous = allocation; + } - allocation->previous = NULL; + allocation->previous = NULL; #endif - fiber_pool->allocations = allocation; - fiber_pool->vacancies = vacancies; - fiber_pool->count += count; - } - RB_VM_LOCK_LEAVE(); + fiber_pool->allocations = allocation; + fiber_pool->vacancies = vacancies; + fiber_pool->count += count; return allocation; } @@ -614,7 +627,9 @@ fiber_pool_initialize(struct fiber_pool * fiber_pool, size_t size, size_t count, fiber_pool->vm_stack_size = vm_stack_size; - fiber_pool_expand(fiber_pool, count); + if (RB_UNLIKELY(!fiber_pool_expand(fiber_pool, count))) { + rb_raise(rb_eFiberError, "can't allocate initial fiber stacks (%"PRIuSIZE" x %"PRIuSIZE" bytes): %s", count, fiber_pool->size, strerror(errno)); + } } #ifdef FIBER_POOL_ALLOCATION_FREE @@ -662,31 +677,79 @@ fiber_pool_allocation_free(struct fiber_pool_allocation * allocation) } #endif +// Number of stacks to request when expanding the pool (clamped to min/max). +static inline size_t +fiber_pool_stack_expand_count(const struct fiber_pool *pool) +{ + const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE; + const size_t minimum = pool->initial_count; + + size_t count = pool->count; + if (count > maximum) count = maximum; + if (count < minimum) count = minimum; + + return count; +} + +// When the vacancy list is empty, grow the pool (and run GC only if mmap fails). Caller holds the VM lock. +// Returns NULL if expansion failed after GC + retry; errno is set. Otherwise returns a vacancy. +static struct fiber_pool_vacancy * +fiber_pool_stack_acquire_expand(struct fiber_pool *fiber_pool) +{ + size_t count = fiber_pool_stack_expand_count(fiber_pool); + + if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expanding fiber pool by %"PRIuSIZE" stacks\n", count); + + struct fiber_pool_vacancy *vacancy = NULL; + + if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) { + return fiber_pool_vacancy_pop(fiber_pool); + } + else { + if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: expand failed (%s), collecting garbage\n", strerror(errno)); + + rb_gc(); + + // After running GC, the vacancy list may have some stacks: + vacancy = fiber_pool_vacancy_pop(fiber_pool); + if (RB_LIKELY(vacancy)) { + return vacancy; + } + + // Try to expand the fiber pool again: + if (RB_LIKELY(fiber_pool_expand(fiber_pool, count))) { + return fiber_pool_vacancy_pop(fiber_pool); + } + else { + // Okay, we really failed to acquire a stack. Give up and return NULL with errno set: + return NULL; + } + } +} + // Acquire a stack from the given fiber pool. If none are available, allocate more. static struct fiber_pool_stack fiber_pool_stack_acquire(struct fiber_pool * fiber_pool) { - struct fiber_pool_vacancy * vacancy ; - RB_VM_LOCK_ENTER(); + struct fiber_pool_vacancy * vacancy; + + unsigned int lev; + RB_VM_LOCK_ENTER_LEV(&lev); { + // Fast path: try to acquire a stack from the vacancy list: vacancy = fiber_pool_vacancy_pop(fiber_pool); if (DEBUG) fprintf(stderr, "fiber_pool_stack_acquire: %p used=%"PRIuSIZE"\n", (void*)fiber_pool->vacancies, fiber_pool->used); - if (!vacancy) { - const size_t maximum = FIBER_POOL_ALLOCATION_MAXIMUM_SIZE; - const size_t minimum = fiber_pool->initial_count; - - size_t count = fiber_pool->count; - if (count > maximum) count = maximum; - if (count < minimum) count = minimum; + // Slow path: If the pool has no vacancies, expand first. Only run GC when expansion fails (e.g. mmap), so we can reclaim stacks from dead fibers before retrying: + if (RB_UNLIKELY(!vacancy)) { + vacancy = fiber_pool_stack_acquire_expand(fiber_pool); - fiber_pool_expand(fiber_pool, count); - - // The free list should now contain some stacks: - VM_ASSERT(fiber_pool->vacancies); - - vacancy = fiber_pool_vacancy_pop(fiber_pool); + // If expansion failed, raise an error: + if (RB_UNLIKELY(!vacancy)) { + RB_VM_LOCK_LEAVE_LEV(&lev); + rb_raise(rb_eFiberError, "can't allocate fiber stack: %s", strerror(errno)); + } } VM_ASSERT(vacancy); @@ -705,7 +768,7 @@ fiber_pool_stack_acquire(struct fiber_pool * fiber_pool) fiber_pool_stack_reset(&vacancy->stack); } - RB_VM_LOCK_LEAVE(); + RB_VM_LOCK_LEAVE_LEV(&lev); return vacancy->stack; } diff --git a/lib/rubygems.rb b/lib/rubygems.rb index dbdbd1f5c38d53..d289cab0fd627e 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -1300,10 +1300,17 @@ def register_default_spec(spec) prefix_pattern = /^(#{prefix_group})/ end + native_extension_suffixes = Gem.dynamic_library_suffixes.reject(&:empty?) + spec.files.each do |file| if new_format file = file.sub(prefix_pattern, "") - next unless $~ + unless $~ + # Also register native extension files (e.g. date_core.bundle) + # that are listed without require path prefix in the gemspec + next if file.include?("/") + next unless file.end_with?(*native_extension_suffixes) + end end spec.activate if already_loaded?(file) diff --git a/lib/rubygems/util/licenses.rb b/lib/rubygems/util/licenses.rb index fbb7b55075ea85..caf53d0b7ecb22 100644 --- a/lib/rubygems/util/licenses.rb +++ b/lib/rubygems/util/licenses.rb @@ -27,6 +27,7 @@ class Gem::Licenses AGPL-1.0-or-later AGPL-3.0-only AGPL-3.0-or-later + ALGLIB-Documentation AMD-newlib AMDPLPA AML @@ -48,6 +49,7 @@ class Gem::Licenses Adobe-Display-PostScript Adobe-Glyph Adobe-Utopia + Advanced-Cryptics-Dictionary Afmparse Aladdin Apache-1.0 @@ -61,6 +63,7 @@ class Gem::Licenses Artistic-2.0 Artistic-dist Aspell-RU + BOLA-1.1 BSD-1-Clause BSD-2-Clause BSD-2-Clause-Darwin @@ -80,6 +83,7 @@ class Gem::Licenses BSD-3-Clause-No-Nuclear-Warranty BSD-3-Clause-Open-MPI BSD-3-Clause-Sun + BSD-3-Clause-Tso BSD-3-Clause-acpica BSD-3-Clause-flex BSD-4-Clause @@ -90,6 +94,7 @@ class Gem::Licenses BSD-Advertising-Acknowledgement BSD-Attribution-HPND-disclaimer BSD-Inferno-Nettverk + BSD-Mark-Modifications BSD-Protection BSD-Source-Code BSD-Source-beginning-file @@ -111,9 +116,11 @@ class Gem::Licenses Borceux Brian-Gladman-2-Clause Brian-Gladman-3-Clause + Buddy C-UDA-1.0 CAL-1.0 CAL-1.0-Combined-Work-Exception + CAPEC-tou CATOSL-1.1 CC-BY-1.0 CC-BY-2.0 @@ -231,6 +238,9 @@ class Gem::Licenses EPICS EPL-1.0 EPL-2.0 + ESA-PL-permissive-2.4 + ESA-PL-strong-copyleft-2.4 + ESA-PL-weak-copyleft-2.4 EUDatagrid EUPL-1.0 EUPL-1.1 @@ -304,6 +314,7 @@ class Gem::Licenses HPND-Markus-Kuhn HPND-Netrek HPND-Pbmplus + HPND-SMC HPND-UC HPND-UC-export-US HPND-doc @@ -318,6 +329,7 @@ class Gem::Licenses HPND-sell-variant HPND-sell-variant-MIT-disclaimer HPND-sell-variant-MIT-disclaimer-rev + HPND-sell-variant-critical-systems HTMLTIDY HaskellReport Hippocratic-2.1 @@ -330,6 +342,7 @@ class Gem::Licenses IPL-1.0 ISC ISC-Veillard + ISO-permission ImageMagick Imlib2 Info-ZIP @@ -387,6 +400,7 @@ class Gem::Licenses MIT-Festival MIT-Khronos-old MIT-Modern-Variant + MIT-STK MIT-Wu MIT-advertising MIT-enna @@ -395,6 +409,7 @@ class Gem::Licenses MIT-testregex MITNFA MMIXware + MMPL-1.0.1 MPEG-SSG MPL-1.0 MPL-1.1 @@ -426,6 +441,7 @@ class Gem::Licenses NGPL NICTA-1.0 NIST-PD + NIST-PD-TNT NIST-PD-fallback NIST-Software NLOD-1.0 @@ -485,12 +501,15 @@ class Gem::Licenses OPL-1.0 OPL-UK-3.0 OPUBL-1.0 + OSC-1.0 OSET-PL-2.1 OSL-1.0 OSL-1.1 OSL-2.0 OSL-2.1 OSL-3.0 + OSSP + OpenMDW-1.0 OpenPBS-2.3 OpenSSL OpenSSL-standalone @@ -501,6 +520,7 @@ class Gem::Licenses PHP-3.01 PPL PSF-2.0 + ParaType-Free-Font-1.3 Parity-6.0.0 Parity-7.0.0 Pixar @@ -529,6 +549,7 @@ class Gem::Licenses SGI-B-1.1 SGI-B-2.0 SGI-OpenGL + SGMLUG-PM SGP4 SHL-0.5 SHL-0.51 @@ -576,6 +597,7 @@ class Gem::Licenses TTYP0 TU-Berlin-1.0 TU-Berlin-2.0 + TekHVC TermReadKey ThirdEye TrustedQSL @@ -585,6 +607,7 @@ class Gem::Licenses UPL-1.0 URT-RLE Ubuntu-font-1.0 + UnRAR Unicode-3.0 Unicode-DFS-2015 Unicode-DFS-2016 @@ -596,15 +619,19 @@ class Gem::Licenses VOSTROM VSL-1.0 Vim + Vixie-Cron W3C W3C-19980720 W3C-20150513 + WTFNMFPL WTFPL Watcom-1.0 Widget-Workshop + WordNet Wsuipa X11 X11-distribute-modifications-variant + X11-no-permit-persons X11-swapped XFree86-1.1 XSkat @@ -645,6 +672,7 @@ class Gem::Licenses gnuplot gtkbook hdparm + hyphen-bulgarian iMatix jove libpng-1.6.35 @@ -734,6 +762,7 @@ class Gem::Licenses CGAL-linking-exception CLISP-exception-2.0 Classpath-exception-2.0 + Classpath-exception-2.0-short DigiRule-FOSS-exception Digia-Qt-LGPL-exception-1.1 FLTK-exception @@ -775,6 +804,7 @@ class Gem::Licenses SHL-2.0 SHL-2.1 SWI-exception + Simple-Library-Usage-exception Swift-exception Texinfo-exception UBDL-exception @@ -788,12 +818,15 @@ class Gem::Licenses gnu-javamail-exception harbour-exception i2p-gpl-java-exception + kvirc-openssl-exception libpri-OpenH323-exception mif-exception mxml-exception openvpn-openssl-exception polyparse-exception romic-exception + rsync-linking-exception + sqlitestudio-OpenSSL-exception stunnel-exception u-boot-exception-2.0 vsftpd-openssl-exception diff --git a/test/ruby/test_fiber.rb b/test/ruby/test_fiber.rb index b7d2b71c196c38..dc48d9219f9919 100644 --- a/test/ruby/test_fiber.rb +++ b/test/ruby/test_fiber.rb @@ -506,4 +506,22 @@ def test_machine_stack_gc GC.start RUBY end + + def test_fiber_pool_stack_acquire_failure + omit "cannot determine max_map_count" unless File.exist?("/proc/sys/vm/max_map_count") + # On these platforms, excessive memory usage can cause the test to fail unexpectedly. + omit "not supported on IBM platforms" if RUBY_PLATFORM =~ /s390x|powerpc/ + omit "not supported with YJIT" if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled? + omit "not supported with ZJIT" if defined?(RubyVM::ZJIT) && RubyVM::ZJIT.enabled? + + assert_separately([], <<~RUBY, timeout: 120) + max_map_count = File.read("/proc/sys/vm/max_map_count").to_i + GC.disable + assert_nothing_raised do + (max_map_count + 10).times do + Fiber.new { Fiber.yield }.resume + end + end + RUBY + end end diff --git a/test/ruby/test_thread.rb b/test/ruby/test_thread.rb index d43c31ce09044a..2dea3359c185bb 100644 --- a/test/ruby/test_thread.rb +++ b/test/ruby/test_thread.rb @@ -1682,7 +1682,10 @@ def self.make_finalizer(stdin, stdout, stderr, wait_thread) stdin.close rescue nil stdout.close rescue nil stderr.close rescue nil - wait_thread.value + # On some GC implementations (e.g. mmtk), finalizers run as postponed + # jobs which can execute on any thread, including the wait_thread itself. + # Guard against joining the current thread. + wait_thread.value unless Thread.current == wait_thread end end end diff --git a/test/rubygems/helper.rb b/test/rubygems/helper.rb index b274273069560a..9ddc79e6e841b2 100644 --- a/test/rubygems/helper.rb +++ b/test/rubygems/helper.rb @@ -2,13 +2,6 @@ require "rubygems" -begin - gem "test-unit", "~> 3.0" -rescue Gem::LoadError -end - -require "test/unit" - begin raise LoadError if ENV["GEM_COMMAND"] @@ -35,6 +28,13 @@ # SimpleCov is not installed end +begin + gem "test-unit", "~> 3.0" +rescue Gem::LoadError +end + +require "test/unit" + require "fileutils" require "pathname" require "pp" diff --git a/test/rubygems/test_gem.rb b/test/rubygems/test_gem.rb index 4293bc5ba81734..de475d0891fd3f 100644 --- a/test/rubygems/test_gem.rb +++ b/test/rubygems/test_gem.rb @@ -1655,6 +1655,27 @@ def test_register_default_spec assert_nil Gem.find_unresolved_default_spec("README") end + def test_register_default_spec_new_style_with_native_extension + Gem.clear_default_specs + + dlext = RbConfig::CONFIG["DLEXT"] + + new_style = Gem::Specification.new do |spec| + spec.name = "my_ext" + spec.version = "1.0" + spec.files = ["lib/my_ext.rb", "my_ext_core.#{dlext}", "ext/my_ext/my_ext_core.c", "README.md"] + spec.require_paths = ["lib"] + end + + Gem.register_default_spec new_style + + assert_equal new_style, Gem.find_unresolved_default_spec("my_ext.rb") + assert_equal new_style, Gem.find_unresolved_default_spec("my_ext_core") + assert_equal new_style, Gem.find_unresolved_default_spec("my_ext_core.#{dlext}") + assert_nil Gem.find_unresolved_default_spec("ext/my_ext/my_ext_core.c") + assert_nil Gem.find_unresolved_default_spec("README.md") + end + def test_register_default_spec_old_style_with_folder_starting_with_lib Gem.clear_default_specs diff --git a/test/rubygems/test_gem_commands_owner_command.rb b/test/rubygems/test_gem_commands_owner_command.rb index e48410ea9cd6ed..cf0fe521a21518 100644 --- a/test/rubygems/test_gem_commands_owner_command.rb +++ b/test/rubygems/test_gem_commands_owner_command.rb @@ -399,6 +399,7 @@ def test_with_webauthn_enabled_success end def test_with_webauthn_enabled_failure + pend "Flaky on TruffleRuby" if RUBY_ENGINE == "truffleruby" response_success = "Owner added successfully." server = Gem::MockTCPServer.new error = Gem::WebauthnVerificationError.new("Something went wrong") diff --git a/tool/test-bundled-gems.rb b/tool/test-bundled-gems.rb index a46052bff578d9..b603cc09d7a530 100644 --- a/tool/test-bundled-gems.rb +++ b/tool/test-bundled-gems.rb @@ -2,8 +2,10 @@ require 'timeout' require 'fileutils' require 'shellwords' +require 'etc' require_relative 'lib/colorize' require_relative 'lib/gem_env' +require_relative 'lib/test/jobserver' ENV.delete("GNUMAKEFLAGS") @@ -28,6 +30,20 @@ exit_code = 0 ruby = ENV['RUBY'] || RbConfig.ruby failed = [] + +max = ENV['TEST_BUNDLED_GEMS_NPROCS']&.to_i || [Etc.nprocessors, 8].min +nprocs = Test::JobServer.max_jobs(max) || max +nprocs = 1 if nprocs < 1 + +if /mingw|mswin/ =~ RUBY_PLATFORM + spawn_group = :new_pgroup + signal_prefix = "" +else + spawn_group = :pgroup + signal_prefix = "-" +end + +jobs = [] File.foreach("#{gem_dir}/bundled_gems") do |line| next unless gem = line[/^[^\s\#]+/] next if bundled_gems&.none? {|pat| File.fnmatch?(pat, gem)} @@ -35,6 +51,7 @@ test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", rake, "test"] first_timeout = 600 # 10min + env_rubylib = rubylib toplib = gem unless File.exist?("#{gem_dir}/src/#{gem}/lib/#{toplib}.rb") @@ -68,7 +85,9 @@ # Since debug gem requires debug.so in child processes without # activating the gem, we preset necessary paths in RUBYLIB # environment variable. - load_path = true + libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) + next unless $?.success? + env_rubylib = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) when "test-unit" test_command = [ruby, *run_opts, "-C", "#{gem_dir}/src/#{gem}", "test/run.rb"] @@ -81,66 +100,144 @@ end - if load_path - libs = IO.popen([ruby, "-e", "old = $:.dup; require '#{toplib}'; puts $:-old"], &:read) - next unless $?.success? - ENV["RUBYLIB"] = [libs.split("\n"), rubylib].join(File::PATH_SEPARATOR) - else - ENV["RUBYLIB"] = rubylib - end + jobs << { + gem: gem, + test_command: test_command, + first_timeout: first_timeout, + rubylib: env_rubylib, + } +end + +running_pids = [] +interrupted = false - print (github_actions ? "::group::" : "\n") - puts colorize.decorate("Testing the #{gem} gem", "note") - print "[command]" if github_actions - p test_command - start_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) - timeouts = {nil => first_timeout, INT: 30, TERM: 10, KILL: nil} - if /mingw|mswin/ =~ RUBY_PLATFORM - timeouts.delete(:TERM) # Inner process signal on Windows - group = :new_pgroup - pg = "" - else - group = :pgroup - pg = "-" +trap(:INT) do + interrupted = true + running_pids.each do |pid| + Process.kill("#{signal_prefix}INT", pid) rescue nil end - pid = Process.spawn(*test_command, group => true) - timeouts.each do |sig, sec| - if sig - puts "Sending #{sig} signal" - Process.kill("#{pg}#{sig}", pid) - end - begin - break Timeout.timeout(sec) {Process.wait(pid)} - rescue Timeout::Error +end + +results = Array.new(jobs.size) +queue = Queue.new +jobs.each_with_index { |j, i| queue << [j, i] } +nprocs.times { queue << nil } +print_queue = Queue.new + +puts "Running #{jobs.size} gem tests with #{nprocs} workers..." + +printer = Thread.new do + printed = 0 + while printed < jobs.size + result = print_queue.pop + break if result.nil? + + gem = result[:gem] + elapsed = result[:elapsed] + status = result[:status] + t = " in %.6f sec" % elapsed + + print (github_actions ? "::group::" : "\n") + puts colorize.decorate("Testing the #{gem} gem", "note") + print "[command]" if github_actions + p result[:test_command] + result[:log_lines].each { |l| puts l } + print result[:output] + print "::endgroup::\n" if github_actions + + if status&.success? + puts colorize.decorate("Test passed#{t}", "pass") + else + mesg = "Tests failed " + + (status&.signaled? ? "by SIG#{Signal.signame(status.termsig)}" : + "with exit code #{status&.exitstatus}") + t + puts colorize.decorate(mesg, "fail") + if allowed_failures.include?(gem) + mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" + puts colorize.decorate(mesg, "skip") + else + failed << gem + exit_code = 1 + end end - rescue Interrupt - exit_code = Signal.list["INT"] - Process.kill("#{pg}KILL", pid) - Process.wait(pid) - break + + printed += 1 end +end - elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_at - print "::endgroup::\n" if github_actions - - t = " in %.6f sec" % elapsed - - if $?.success? - puts colorize.decorate("Test passed#{t}", "pass") - else - mesg = "Tests failed " + - ($?.signaled? ? "by SIG#{Signal.signame($?.termsig)}" : - "with exit code #{$?.exitstatus}") + t - puts colorize.decorate(mesg, "fail") - if allowed_failures.include?(gem) - mesg = "Ignoring test failures for #{gem} due to \$TEST_BUNDLED_GEMS_ALLOW_FAILURES or DEFAULT_ALLOWED_FAILURES" - puts colorize.decorate(mesg, "skip") - else - failed << gem - exit_code = 1 +threads = nprocs.times.map do + Thread.new do + while (item = queue.pop) + break if interrupted + job, index = item + + start_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + rd, wr = IO.pipe + env = { "RUBYLIB" => job[:rubylib] } + pid = Process.spawn(env, *job[:test_command], spawn_group => true, [:out, :err] => wr) + wr.close + running_pids << pid + output_thread = Thread.new { rd.read } + + timeouts = { nil => job[:first_timeout], INT: 30, TERM: 10, KILL: nil } + if /mingw|mswin/ =~ RUBY_PLATFORM + timeouts.delete(:TERM) + end + + log_lines = [] + status = nil + timeouts.each do |sig, sec| + if sig + log_lines << "Sending #{sig} signal" + begin + Process.kill("#{signal_prefix}#{sig}", pid) + rescue Errno::ESRCH + _, status = Process.wait2(pid) unless status + break + end + end + begin + break Timeout.timeout(sec) { _, status = Process.wait2(pid) } + rescue Timeout::Error + end + end + + captured = output_thread.value + rd.close + running_pids.delete(pid) + + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_at + + result = { + gem: job[:gem], + test_command: job[:test_command], + status: status, + elapsed: elapsed, + output: captured, + log_lines: log_lines, + } + results[index] = result + print_queue << result end end end -puts "Failed gems: #{failed.join(', ')}" unless failed.empty? +threads.each(&:join) +print_queue << nil +printer.join + +if interrupted + exit Signal.list["INT"] +end + +unless failed.empty? + puts "\n#{colorize.decorate("Failed gems: #{failed.join(', ')}", "fail")}" + results.compact.each do |result| + next if result[:status]&.success? + next if allowed_failures.include?(result[:gem]) + puts colorize.decorate("\nTesting the #{result[:gem]} gem", "note") + print result[:output] + end +end exit exit_code diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 703901b6bc30f4..f897347f87cd7e 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -5288,3 +5288,28 @@ fn test_polymorphic_getivar_too_complex_shape() { [normal.foo, complex.foo] "#), @"[:normal, :complex]"); } + +/// When a method with keyword defaults contains a block that creates a lambda, +/// the lambda causes EP escape, which globally patches NoEPEscape PatchPoints. +/// On subsequent calls the PatchPoint side exit (which uses without_locals()) +/// must not leave stale keyword default values in the frame. We solve this by +/// invalidating the ISEQ version on EP escape so the interpreter takes over. +#[test] +fn test_ep_escape_preserves_keyword_default() { + set_call_threshold(1); + assert_snapshot!(inspect(r#" + def target(dumped, additional_methods: []) + dumped.class + additional_methods.each { |m| ->{ m } } + additional_methods + end + + def forwarder(x, **kwargs) + target(x, **kwargs) + end + + 5.times { forwarder("z") } + forwarder("y", additional_methods: [:to_s]) + target("x") + "#), @"[]"); +} diff --git a/zjit/src/invariants.rs b/zjit/src/invariants.rs index c7dad37b5e8e5f..40768c05fa009d 100644 --- a/zjit/src/invariants.rs +++ b/zjit/src/invariants.rs @@ -223,6 +223,25 @@ pub extern "C" fn rb_zjit_invalidate_no_ep_escape(iseq: IseqPtr) { let cb = ZJITState::get_code_block(); compile_patch_points!(cb, patch_points, EP, "EP is escaped: {}", iseq_name(iseq)); + // Also invalidate the ISEQ version so the method falls back to the + // interpreter on the next call. NoEPEscape PatchPoint side exits use + // without_locals() and don't save locals to the frame. If a PatchPoint + // fires on a later call (where EP hasn't escaped), the interpreter would + // read stale locals (e.g., nil instead of [] for keyword defaults). + // + // We can't use invalidate_iseq_version() here because it skips when + // at MAX_ISEQ_VERSIONS (to prevent unbounded recompilation). Instead, + // directly mark the version as invalidated and reset jit_func so the + // interpreter takes over permanently. + let payload = crate::payload::get_or_create_iseq_payload(iseq); + if let Some(version) = payload.versions.last_mut() { + use crate::payload::IseqStatus; + if unsafe { version.as_ref() }.status != IseqStatus::Invalidated { + unsafe { version.as_mut() }.status = IseqStatus::Invalidated; + unsafe { rb_iseq_reset_jit_func(iseq) }; + } + } + cb.mark_all_executable(); } });