From 55320e5daeb87d9890b743d7c827b7c6c2e7ec71 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 18 Oct 2026 11:14:07 +0900 Subject: [PATCH 1/3] Sample memory usage trends in leak assertions Repeated sampling avoids false leak reports from transient RSS growth. --- lib/core_assertions.rb | 77 ++++++++++++++++++++++++++++++------ test/test_core_assertions.rb | 24 +++++++++++ 2 files changed, 88 insertions(+), 13 deletions(-) diff --git a/lib/core_assertions.rb b/lib/core_assertions.rb index cbc4c05..4d6af56 100644 --- a/lib/core_assertions.rb +++ b/lib/core_assertions.rb @@ -166,7 +166,30 @@ def syntax_check(code, fname, line) end end - def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, **opt) + private def memory_growth_rate(points) + n_1 = (n = points.length) - 1 + mean_x = n_1 / 2.0 + mean_y = points.inject(:+).fdiv(n) + covariance = 0 + points.each_with_index {|value, i| covariance += (i - mean_x) * (value - mean_y)} + variance = (n_1 * n * (2 * n - 1) / 6) - (n_1 * n * mean_x) + (n * mean_x**2) + covariance / variance + end + + private def memory_growth_rates(points) + midpoint = (points.length - 1) / 2 + [memory_growth_rate(points[0..midpoint]), memory_growth_rate(points[midpoint..-1])] + end + + private def memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + if early_rate <= 0 + late_rate <= 0 + else + late_rate <= early_rate * growth_rate_ratio + end + end + + def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, samples: 1, growth_rate_ratio: 0.25, **opt) # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? # For previous versions which implemented MJIT @@ -192,23 +215,51 @@ def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: fal envs = newenvs if newenvs end args.unshift(envs) if envs - cmd = [ - 'END {STDERR.puts '"#{token_dump}"'"FINAL=#{Memory::Status.new}"}', - prepare, - 'STDERR.puts('"#{token_dump}"'"START=#{$initial_status = Memory::Status.new}")', - '$initial_size = $initial_status.size', - code, - 'GC.start', - ].join("\n") + unless Integer === samples and samples > 0 + raise ArgumentError, "samples must be a positive integer" + end + unless (0..1).cover?(growth_rate_ratio) + raise ArgumentError, "growth_rate_ratio must be between 0 and 1" + end + + cmd = <<~RUBY + END {STDERR.puts #{token_dump}"FINAL=\#{Memory::Status.new}"} + #{prepare} + STDERR.puts(#{token_dump}"START=\#{$initial_status = Memory::Status.new}") + $initial_size = $initial_status.size + #{samples}.times do + begin + #{code.gsub(/^/, ' ')} + ensure + GC.start + STDERR.puts(#{token_dump}"SAMPLE=\#{Memory::Status.new}") + end + end + RUBY _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) + sample_statuses = [] + err.gsub!(/^#{token_re}SAMPLE=(\{.*\})\n/) {sample_statuses << Memory::Status.parse($1); ''} assert(status.success?, FailDesc[status, message, err]) + assert_equal(samples, sample_statuses.length, "could not collect all memory samples") ([:size, (rss && :rss)] & after.members).each do |n| - b = before[n] - a = after[n] - next unless a > 0 and b > 0 - assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + if samples == 1 + b = before[n] + a = after[n] + next unless a > 0 and b > 0 + assert_operator(a.fdiv(b), :<, limit, message(message) {"#{n}: #{b} => #{a}"}) + else + values = [before[n], *sample_statuses.map {|sample| sample[n]}] + next unless values.all? {|value| value && value > 0} + + early_rate, late_rate = memory_growth_rates(values) + settled = memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + assert(settled, message(message) { + "#{n} growth did not settle: #{values.inspect} " \ + "(early rate: #{early_rate}, late rate: #{late_rate})" + }) + end end rescue LoadError pend diff --git a/test/test_core_assertions.rb b/test/test_core_assertions.rb index 8293c81..43962b8 100644 --- a/test/test_core_assertions.rb +++ b/test/test_core_assertions.rb @@ -28,6 +28,30 @@ def test_message_adds_sentence_endings assert_equal("details.\ndefault.", message.call) end + def test_memory_growth_is_not_settled_by_a_small_slope_decrease + samples = [ + 19_759_104, 40_828_928, 49_414_144, 59_277_312, 68_714_496, + 77_955_072, 87_113_728, 96_468_992, 105_005_056, 113_754_112, + 122_568_704, + ] + + early_rate, late_rate = send(:memory_growth_rates, samples) + + assert_not_send([self, :memory_growth_settled?, early_rate, late_rate, 0.25]) + assert_send([self, :memory_growth_settled?, early_rate, late_rate, 0.9]) + end + + def test_memory_growth_is_settled_after_growth_stops + early_rate, late_rate = send(:memory_growth_rates, [0, 10, 20, 30, 30, 30, 30, 30]) + + assert_send([self, :memory_growth_settled?, early_rate, late_rate, 0.25]) + end + + def test_memory_growth_ratio_zero_requires_nonincreasing_usage + assert_send([self, :memory_growth_settled?, 10.0, 0.0, 0.0]) + assert_not_send([self, :memory_growth_settled?, 10.0, 0.1, 0.0]) + end + def test_assert_separately_runs_assertions_in_child_ruby assert_separately([], <<~RUBY) assert_equal(4, 2 + 2) From ab24a357a4d8bb5185d4915c6ceb9e4147b371be Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Sun, 18 Oct 2026 19:11:50 +0900 Subject: [PATCH 2/3] Support adaptive memory leak samples Allow memory leak assertions to stop sampling when memory growth settles. This helps avoid failures from transient memory use. --- lib/core_assertions.rb | 135 +++++++++++++++++++++++++++++++---- test/test_core_assertions.rb | 59 +++++++++++++++ 2 files changed, 182 insertions(+), 12 deletions(-) diff --git a/lib/core_assertions.rb b/lib/core_assertions.rb index 4d6af56..1e4a67a 100644 --- a/lib/core_assertions.rb +++ b/lib/core_assertions.rb @@ -181,6 +181,18 @@ def syntax_check(code, fname, line) [memory_growth_rate(points[0..midpoint]), memory_growth_rate(points[midpoint..-1])] end + private def memory_growth_tail_rates(points) + window = [points.length / 4, 2].max + first = points[[points.length - window * 2, 0].max, window] + last = points[-window, window] + [memory_growth_rate(first), memory_growth_rate(last)] + end + + private def memory_growth_settled_in_tail?(points, growth_rate_ratio) + early_rate, late_rate = memory_growth_tail_rates(points) + memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + end + private def memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) if early_rate <= 0 late_rate <= 0 @@ -189,6 +201,9 @@ def syntax_check(code, fname, line) end end + # growth_rate_ratio compares late and early memory growth rates. With a + # sample range, sampling stops when the late rate is at most this ratio + # of the early rate, after collecting the minimum number of samples. def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: false, samples: 1, growth_rate_ratio: 0.25, **opt) # TODO: consider choosing some appropriate limit for RJIT and stop skipping this once it does not randomly fail pend 'assert_no_memory_leak may consider RJIT memory usage as leak' if defined?(RubyVM::RJIT) && RubyVM::RJIT.enabled? @@ -202,7 +217,28 @@ def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: fal raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) token_dump, token_re = new_test_token + start_re = /^#{token_re}START=(\{.*\})\n/ + sample_re = /^#{token_re}SAMPLE=(\{.*\})\n/ + final_re = /^#{token_re}FINAL=(\{.*\})\n/ envs = args.shift if Array === args and Hash === args.first + case samples + when Integer + unless samples > 0 + raise ArgumentError, "samples must be a positive integer or a bounded range" + end + min_samples = max_samples = samples + adaptive_samples = false + when Range + min_samples = samples.begin + max_samples = samples.end + max_samples -= 1 if samples.exclude_end? && Integer === max_samples + unless Integer === min_samples and Integer === max_samples and min_samples >= 2 and max_samples >= min_samples + raise ArgumentError, "samples range must have a minimum of 2 and a bounded positive maximum" + end + adaptive_samples = true + else + raise ArgumentError, "samples must be a positive integer or a bounded range" + end args = [ "--disable=gems", "-r", File.expand_path("../memory_status", __FILE__), @@ -215,36 +251,106 @@ def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: fal envs = newenvs if newenvs end args.unshift(envs) if envs - unless Integer === samples and samples > 0 - raise ArgumentError, "samples must be a positive integer" - end unless (0..1).cover?(growth_rate_ratio) raise ArgumentError, "growth_rate_ratio must be between 0 and 1" end cmd = <<~RUBY + STDERR.sync = true + #{<<~ADAPTIVE if adaptive_samples} + memory_leak_control = STDIN.dup + STDIN.reopen(File::NULL, "r") + ADAPTIVE END {STDERR.puts #{token_dump}"FINAL=\#{Memory::Status.new}"} #{prepare} STDERR.puts(#{token_dump}"START=\#{$initial_status = Memory::Status.new}") $initial_size = $initial_status.size - #{samples}.times do + #{max_samples}.times do begin #{code.gsub(/^/, ' ')} ensure GC.start STDERR.puts(#{token_dump}"SAMPLE=\#{Memory::Status.new}") + #{<<~ADAPTIVE if adaptive_samples} + break unless memory_leak_control.gets == "continue\\n" + ADAPTIVE end end + __END__ RUBY - _, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) - before = err.sub!(/^#{token_re}START=(\{.*\})\n/, '') && Memory::Status.parse($1) - after = err.sub!(/^#{token_re}FINAL=(\{.*\})\n/, '') && Memory::Status.parse($1) sample_statuses = [] - err.gsub!(/^#{token_re}SAMPLE=(\{.*\})\n/) {sample_statuses << Memory::Status.parse($1); ''} + before = after = nil + if adaptive_samples + stdout, err, status = EnvUtil.invoke_ruby(args, "", true, true, **opt) do |in_p, out_p, err_p, pid| + out_reader = Thread.new {out_p.read} + in_p.write(cmd) + in_p.flush + err = +"" + timeout = EnvUtil.apply_timeout_scale(opt.fetch(:timeout, 10)) + timed_out = nil + begin + Timeout.timeout(timeout) do + while line = err_p.gets + case line + when start_re + before = Memory::Status.parse($1) + when sample_re + sample_statuses << Memory::Status.parse($1) + count = sample_statuses.length + settled = false + if count >= min_samples && count < max_samples + metrics = [:size, (rss && :rss)] & sample_statuses.last.members + settled = metrics.all? do |n| + values = [before[n], *sample_statuses.map {|sample| sample[n]}] + next true unless values.all? {|value| value && value > 0} + + memory_growth_settled_in_tail?(values, growth_rate_ratio) + end + end + if settled || count == max_samples + in_p.close + else + in_p.write("continue\n") + in_p.flush + end + when final_re + after = Memory::Status.parse($1) + else + err << line + end + end + in_p.close + status = Process.wait2(pid)[1] + end + rescue Timeout::Error + timed_out = Time.now + status = begin + EnvUtil.terminate(pid, opt.fetch(:signal, :TERM), opt[:pgroup], opt.fetch(:reprieve, 1)) + rescue Errno::ESRCH, Errno::ECHILD + warn "child not found: #{pid}: #$!" + end + in_p.close + end + stdout = out_reader.value + if timed_out + message = "execution of assert_no_memory_leak expired timeout (#{timeout} sec)" + message = EnvUtil.failure_description(status, timed_out, message, [stdout, err].join("\n")) if status + raise opt.fetch(:timeout_error, Timeout::Error), message + end + [stdout, err, status] + end + else + stdout, err, status = EnvUtil.invoke_ruby(args, cmd, true, true, **opt) + before = err.sub!(start_re, '') && Memory::Status.parse($1) + err.gsub!(sample_re) {sample_statuses << Memory::Status.parse($1); ''} + after = err.sub!(final_re, '') && Memory::Status.parse($1) + end assert(status.success?, FailDesc[status, message, err]) - assert_equal(samples, sample_statuses.length, "could not collect all memory samples") + assert_operator(sample_statuses.length, :>=, min_samples, "could not collect the minimum memory samples") + assert_operator(sample_statuses.length, :<=, max_samples, "collected more than the maximum memory samples") + assert_equal(samples, sample_statuses.length, "could not collect all memory samples") unless adaptive_samples ([:size, (rss && :rss)] & after.members).each do |n| - if samples == 1 + if max_samples == 1 b = before[n] a = after[n] next unless a > 0 and b > 0 @@ -253,8 +359,13 @@ def assert_no_memory_leak(args, prepare, code, message=nil, limit: 2.0, rss: fal values = [before[n], *sample_statuses.map {|sample| sample[n]}] next unless values.all? {|value| value && value > 0} - early_rate, late_rate = memory_growth_rates(values) - settled = memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + if adaptive_samples + early_rate, late_rate = memory_growth_tail_rates(values) + settled = memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + else + early_rate, late_rate = memory_growth_rates(values) + settled = memory_growth_settled?(early_rate, late_rate, growth_rate_ratio) + end assert(settled, message(message) { "#{n} growth did not settle: #{values.inspect} " \ "(early rate: #{early_rate}, late rate: #{late_rate})" diff --git a/test/test_core_assertions.rb b/test/test_core_assertions.rb index 43962b8..197bbc7 100644 --- a/test/test_core_assertions.rb +++ b/test/test_core_assertions.rb @@ -47,11 +47,70 @@ def test_memory_growth_is_settled_after_growth_stops assert_send([self, :memory_growth_settled?, early_rate, late_rate, 0.25]) end + def test_memory_growth_tail_is_settled_after_a_late_step + samples = [0, 10, 20, 30, 30, 30, 30, 30, 40, 50, 60, 60, 60, 60, 60] + + assert_send([self, :memory_growth_settled_in_tail?, samples, 0.25]) + end + def test_memory_growth_ratio_zero_requires_nonincreasing_usage assert_send([self, :memory_growth_settled?, 10.0, 0.0, 0.0]) assert_not_send([self, :memory_growth_settled?, 10.0, 0.1, 0.0]) end + def test_assert_no_memory_leak_stops_at_minimum_sample_count_when_settled + prepare = <<~'RUBY' + $memory_sample_count = 0 + END {raise "unexpected sample count" unless $memory_sample_count == 2} + class FakeMemoryStatus + attr_reader :size + def initialize(size) @size = size end + def to_s; "{#{Memory::Status.members.map {|member| "#{member}:#{size}"}.join(',')}}" end + end + Memory::Status.define_singleton_method(:new) {FakeMemoryStatus.new(1)} + RUBY + code = '$memory_sample_count += 1; raise "unexpected stdin" unless STDIN.gets.nil?' + + assert_no_memory_leak([], prepare, code, samples: 2..5, growth_rate_ratio: 1.0) + end + + def test_assert_no_memory_leak_fails_at_maximum_sample_count_when_growth_continues + prepare = <<~'RUBY' + $memory_sample_count = 0 + $memory_status_count = 0 + END {raise "unexpected sample count" unless $memory_sample_count == 4} + class FakeMemoryStatus + attr_reader :size + def initialize(size) @size = size end + def to_s; "{#{Memory::Status.members.map {|member| "#{member}:#{size}"}.join(',')}}" end + end + Memory::Status.define_singleton_method(:new) do + size = 100 + $memory_status_count * 10 + $memory_status_count += 1 + FakeMemoryStatus.new(size) + end + RUBY + code = '$memory_sample_count += 1' + + error = assert_raise(Test::Unit::AssertionFailedError) do + assert_no_memory_leak([], prepare, code, samples: 2..4, growth_rate_ratio: 0.25) + end + assert_match(/growth did not settle/, error.message) + end + + def test_assert_no_memory_leak_rejects_ranges_without_enough_samples + error = assert_raise(ArgumentError) do + assert_no_memory_leak([], "", "", samples: 1..5) + end + assert_match(/minimum of 2/, error.message) + end + + def test_assert_no_memory_leak_applies_timeout_to_sample_range + assert_raise(Timeout::Error) do + assert_no_memory_leak([], "", "sleep", samples: 2..5, timeout: 0.1) + end + end + def test_assert_separately_runs_assertions_in_child_ruby assert_separately([], <<~RUBY) assert_equal(4, 2 + 2) From c1f817fdadd4a0520c569f5d02d1fd4d6b2a896e Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Mon, 19 Oct 2026 09:59:03 +0900 Subject: [PATCH 3/3] Wait indefinitely for dumped child processes On macOS, it sometimes takes a little while for a process attached to the debugger to be ready for `waitpid`. --- lib/envutil.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/envutil.rb b/lib/envutil.rb index 5716bf7..608ce24 100644 --- a/lib/envutil.rb +++ b/lib/envutil.rb @@ -204,7 +204,7 @@ def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) Process.wait(pid, Process::WNOHANG) rescue nil break end - if signals.empty? or !reprieve + if signals.empty? or !reprieve or dumped Process.wait(pid) else begin