NcamGnrvngu/gitlab-build-images
1
0
Fork 0
mirror of https://ops.gitlab.net/gitlab-org/gitlab-build-images.git synced 2025-12-09 10:02:56 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Consolidated Ruby patches using major and minor versions

Browse source 
Previously we had to remember to apply specific patches for each
version of Ruby, and it was easy to forget. Let's just simplify this
by applying patches for the major and minor versions (e.g. 2.7, 3.0,
etc.).
This commit is contained in:
Stan Hu 2022-11-29 14:49:23 -08:00
parent f0b78cb928
commit 88901ea9ff
No known key found for this signature in database
GPG key ID: 8D3931AD39CC7A20
15 changed files with 4 additions and 1500 deletions
Download patch file Download diff file Expand all files Collapse all files

25
patches/ruby/2.7.4/debug-segfault.patch
View file

@ -1,25 +0,0 @@
diff --git a/class.c b/class.c
index c866d1d727..37ff3c5ade 100644
--- a/class.c
+++ b/class.c
@@ -27,6 +27,7 @@
#include "ruby/st.h"
#include "constant.h"
#include "vm_core.h"
+#include "vm_debug.h"
#include "id_table.h"
#include <ctype.h>
@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg)
while (cur) {
VALUE curklass = cur->klass;
cur = cur->next;
+
+ if (curklass == 0) {
+ fprintf(stderr, "=== Detected NULL subclass:\n");
+ dp(curklass);
+ }
+
f(curklass, arg);
}
}

258
patches/ruby/2.7.4/thread-memory-allocations-2.7.patch
View file

@ -1,258 +0,0 @@
From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 22 Dec 2020 15:33:08 +0100
Subject: [PATCH] Expose `Thread#memory_allocations` counters
This provides currently a per-thread GC heap slots
and malloc allocations statistics.
This is designed to measure a memory allocations
in a multi-threaded environments (concurrent requests
processing) with an accurate information about allocated
memory within a given execution context.
Example: Measure memory pressure generated by a given
requests to easier find requests with a lot of allocations.
Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530
---
gc.c | 20 ++++++
.../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++
thread.c | 55 +++++++++++++++
vm_core.h | 17 +++++
4 files changed, 159 insertions(+)
create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb
diff --git a/gc.c b/gc.c
index 73faf46b128b..f2dcd2935052 100644
--- a/gc.c
+++ b/gc.c
@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
+ }
+#endif
+
objspace->total_allocated_objects++;
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ if (new_size > old_size) {
+ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
+ }
+
+ if (type == MEMOP_TYPE_MALLOC) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
+ }
+ }
+#endif
+
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) {
diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb
new file mode 100644
index 000000000000..2e281513578b
--- /dev/null
+++ b/test/ruby/test_thread_trace_memory_allocations.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+
+class TestThreadTraceMemoryAllocations < Test::Unit::TestCase
+ def test_disabled_trace_memory_allocations
+ Thread.trace_memory_allocations = false
+
+ assert_predicate Thread.current.memory_allocations, :nil?
+ end
+
+ def test_enabled_trace_memory_allocations
+ Thread.trace_memory_allocations = true
+
+ assert_not_nil(Thread.current.memory_allocations)
+ end
+
+ def test_only_this_thread_allocations_are_counted
+ changed = {
+ total_allocated_objects: 1000,
+ total_malloc_bytes: 1_000_000,
+ total_mallocs: 100
+ }
+
+ Thread.trace_memory_allocations = true
+
+ assert_less_than(changed) do
+ Thread.new do
+ assert_greater_than(changed) do
+ # This will allocate: 5k objects, 5k mallocs, 5MB
+ allocate(5000, 1000)
+ end
+ end.join
+
+ # This will allocate: 50 objects, 50 mallocs, 500 bytes
+ allocate(50, 10)
+ end
+ end
+
+ private
+
+ def allocate(slots, bytes)
+ Array.new(slots).map do
+ '0' * bytes
+ end
+ end
+
+ def assert_greater_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}")
+ end
+ end
+
+ def assert_less_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}")
+ end
+ end
+end
diff --git a/thread.c b/thread.c
index 708aaa471d99..d68a59e9f2d6 100644
--- a/thread.c
+++ b/thread.c
@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval)
return rb_vm_thread_backtrace_locations(argc, argv, thval);
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+rb_thread_t *
+ruby_threadptr_for_trace_memory_allocations(void)
+{
+ // The order of this checks is important due
+ // to how Ruby VM is initialized
+ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
+ return GET_THREAD();
+ }
+
+ return NULL;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations(VALUE _)
+{
+ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
+{
+ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
+ return val;
+}
+
+static VALUE
+rb_thread_memory_allocations(VALUE self)
+{
+ rb_thread_t *th = rb_thread_ptr(self);
+
+ if (!th->vm->thread_trace_memory_allocations) {
+ return Qnil;
+ }
+
+ VALUE ret = rb_hash_new();
+
+ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
+ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
+ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
+
+ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
+ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
+ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
+
+ return ret;
+}
+#endif
+
/*
* Document-class: ThreadError
*
@@ -5230,6 +5279,12 @@ Init_Thread(void)
rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
rb_define_alias(rb_cThread, "inspect", "to_s");
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
+ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
+#endif
+
rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
"stream closed in another thread");
diff --git a/vm_core.h b/vm_core.h
index 12c3ac377551..63cdf55fa6ed 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -69,6 +69,13 @@
# define VM_INSN_INFO_TABLE_IMPL 2
#endif
+/*
+ * track a per thread memory allocations
+ */
+#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
+# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
+#endif
+
#include "ruby/ruby.h"
#include "ruby/st.h"
@@ -602,6 +609,7 @@ typedef struct rb_vm_struct {
unsigned int running: 1;
unsigned int thread_abort_on_exception: 1;
unsigned int thread_report_on_exception: 1;
+ unsigned int thread_trace_memory_allocations: 1;
unsigned int safe_level_: 1;
int sleeper;
@@ -960,6 +968,14 @@ typedef struct rb_thread_struct {
rb_thread_list_t *join_list;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ struct {
+ size_t total_allocated_objects;
+ size_t total_malloc_bytes;
+ size_t total_mallocs;
+ } memory_allocations;
+#endif
+
union {
struct {
VALUE proc;
@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
+rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
void rb_execution_context_update(const rb_execution_context_t *ec);

25
patches/ruby/2.7.5/debug-segfault.patch
View file

@ -1,25 +0,0 @@
diff --git a/class.c b/class.c
index c866d1d727..37ff3c5ade 100644
--- a/class.c
+++ b/class.c
@@ -27,6 +27,7 @@
#include "ruby/st.h"
#include "constant.h"
#include "vm_core.h"
+#include "vm_debug.h"
#include "id_table.h"
#include <ctype.h>
@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg)
while (cur) {
VALUE curklass = cur->klass;
cur = cur->next;
+
+ if (curklass == 0) {
+ fprintf(stderr, "=== Detected NULL subclass:\n");
+ dp(curklass);
+ }
+
f(curklass, arg);
}
}

258
patches/ruby/2.7.5/thread-memory-allocations-2.7.patch
View file

@ -1,258 +0,0 @@
From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 22 Dec 2020 15:33:08 +0100
Subject: [PATCH] Expose `Thread#memory_allocations` counters
This provides currently a per-thread GC heap slots
and malloc allocations statistics.
This is designed to measure a memory allocations
in a multi-threaded environments (concurrent requests
processing) with an accurate information about allocated
memory within a given execution context.
Example: Measure memory pressure generated by a given
requests to easier find requests with a lot of allocations.
Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530
---
gc.c | 20 ++++++
.../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++
thread.c | 55 +++++++++++++++
vm_core.h | 17 +++++
4 files changed, 159 insertions(+)
create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb
diff --git a/gc.c b/gc.c
index 73faf46b128b..f2dcd2935052 100644
--- a/gc.c
+++ b/gc.c
@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
+ }
+#endif
+
objspace->total_allocated_objects++;
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ if (new_size > old_size) {
+ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
+ }
+
+ if (type == MEMOP_TYPE_MALLOC) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
+ }
+ }
+#endif
+
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) {
diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb
new file mode 100644
index 000000000000..2e281513578b
--- /dev/null
+++ b/test/ruby/test_thread_trace_memory_allocations.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+
+class TestThreadTraceMemoryAllocations < Test::Unit::TestCase
+ def test_disabled_trace_memory_allocations
+ Thread.trace_memory_allocations = false
+
+ assert_predicate Thread.current.memory_allocations, :nil?
+ end
+
+ def test_enabled_trace_memory_allocations
+ Thread.trace_memory_allocations = true
+
+ assert_not_nil(Thread.current.memory_allocations)
+ end
+
+ def test_only_this_thread_allocations_are_counted
+ changed = {
+ total_allocated_objects: 1000,
+ total_malloc_bytes: 1_000_000,
+ total_mallocs: 100
+ }
+
+ Thread.trace_memory_allocations = true
+
+ assert_less_than(changed) do
+ Thread.new do
+ assert_greater_than(changed) do
+ # This will allocate: 5k objects, 5k mallocs, 5MB
+ allocate(5000, 1000)
+ end
+ end.join
+
+ # This will allocate: 50 objects, 50 mallocs, 500 bytes
+ allocate(50, 10)
+ end
+ end
+
+ private
+
+ def allocate(slots, bytes)
+ Array.new(slots).map do
+ '0' * bytes
+ end
+ end
+
+ def assert_greater_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}")
+ end
+ end
+
+ def assert_less_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}")
+ end
+ end
+end
diff --git a/thread.c b/thread.c
index 708aaa471d99..d68a59e9f2d6 100644
--- a/thread.c
+++ b/thread.c
@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval)
return rb_vm_thread_backtrace_locations(argc, argv, thval);
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+rb_thread_t *
+ruby_threadptr_for_trace_memory_allocations(void)
+{
+ // The order of this checks is important due
+ // to how Ruby VM is initialized
+ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
+ return GET_THREAD();
+ }
+
+ return NULL;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations(VALUE _)
+{
+ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
+{
+ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
+ return val;
+}
+
+static VALUE
+rb_thread_memory_allocations(VALUE self)
+{
+ rb_thread_t *th = rb_thread_ptr(self);
+
+ if (!th->vm->thread_trace_memory_allocations) {
+ return Qnil;
+ }
+
+ VALUE ret = rb_hash_new();
+
+ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
+ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
+ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
+
+ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
+ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
+ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
+
+ return ret;
+}
+#endif
+
/*
* Document-class: ThreadError
*
@@ -5230,6 +5279,12 @@ Init_Thread(void)
rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
rb_define_alias(rb_cThread, "inspect", "to_s");
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
+ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
+#endif
+
rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
"stream closed in another thread");
diff --git a/vm_core.h b/vm_core.h
index 12c3ac377551..63cdf55fa6ed 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -69,6 +69,13 @@
# define VM_INSN_INFO_TABLE_IMPL 2
#endif
+/*
+ * track a per thread memory allocations
+ */
+#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
+# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
+#endif
+
#include "ruby/ruby.h"
#include "ruby/st.h"
@@ -602,6 +609,7 @@ typedef struct rb_vm_struct {
unsigned int running: 1;
unsigned int thread_abort_on_exception: 1;
unsigned int thread_report_on_exception: 1;
+ unsigned int thread_trace_memory_allocations: 1;
unsigned int safe_level_: 1;
int sleeper;
@@ -960,6 +968,14 @@ typedef struct rb_thread_struct {
rb_thread_list_t *join_list;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ struct {
+ size_t total_allocated_objects;
+ size_t total_malloc_bytes;
+ size_t total_mallocs;
+ } memory_allocations;
+#endif
+
union {
struct {
VALUE proc;
@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
+rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
void rb_execution_context_update(const rb_execution_context_t *ec);

25
patches/ruby/2.7.6/debug-segfault.patch
View file

@ -1,25 +0,0 @@
diff --git a/class.c b/class.c
index c866d1d727..37ff3c5ade 100644
--- a/class.c
+++ b/class.c
@@ -27,6 +27,7 @@
#include "ruby/st.h"
#include "constant.h"
#include "vm_core.h"
+#include "vm_debug.h"
#include "id_table.h"
#include <ctype.h>
@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg)
while (cur) {
VALUE curklass = cur->klass;
cur = cur->next;
+
+ if (curklass == 0) {
+ fprintf(stderr, "=== Detected NULL subclass:\n");
+ dp(curklass);
+ }
+
f(curklass, arg);
}
}

258
patches/ruby/2.7.6/thread-memory-allocations-2.7.patch
View file

@ -1,258 +0,0 @@
From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 22 Dec 2020 15:33:08 +0100
Subject: [PATCH] Expose `Thread#memory_allocations` counters
This provides currently a per-thread GC heap slots
and malloc allocations statistics.
This is designed to measure a memory allocations
in a multi-threaded environments (concurrent requests
processing) with an accurate information about allocated
memory within a given execution context.
Example: Measure memory pressure generated by a given
requests to easier find requests with a lot of allocations.
Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530
---
gc.c | 20 ++++++
.../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++
thread.c | 55 +++++++++++++++
vm_core.h | 17 +++++
4 files changed, 159 insertions(+)
create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb
diff --git a/gc.c b/gc.c
index 73faf46b128b..f2dcd2935052 100644
--- a/gc.c
+++ b/gc.c
@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
+ }
+#endif
+
objspace->total_allocated_objects++;
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ if (new_size > old_size) {
+ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
+ }
+
+ if (type == MEMOP_TYPE_MALLOC) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
+ }
+ }
+#endif
+
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) {
diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb
new file mode 100644
index 000000000000..2e281513578b
--- /dev/null
+++ b/test/ruby/test_thread_trace_memory_allocations.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+
+class TestThreadTraceMemoryAllocations < Test::Unit::TestCase
+ def test_disabled_trace_memory_allocations
+ Thread.trace_memory_allocations = false
+
+ assert_predicate Thread.current.memory_allocations, :nil?
+ end
+
+ def test_enabled_trace_memory_allocations
+ Thread.trace_memory_allocations = true
+
+ assert_not_nil(Thread.current.memory_allocations)
+ end
+
+ def test_only_this_thread_allocations_are_counted
+ changed = {
+ total_allocated_objects: 1000,
+ total_malloc_bytes: 1_000_000,
+ total_mallocs: 100
+ }
+
+ Thread.trace_memory_allocations = true
+
+ assert_less_than(changed) do
+ Thread.new do
+ assert_greater_than(changed) do
+ # This will allocate: 5k objects, 5k mallocs, 5MB
+ allocate(5000, 1000)
+ end
+ end.join
+
+ # This will allocate: 50 objects, 50 mallocs, 500 bytes
+ allocate(50, 10)
+ end
+ end
+
+ private
+
+ def allocate(slots, bytes)
+ Array.new(slots).map do
+ '0' * bytes
+ end
+ end
+
+ def assert_greater_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}")
+ end
+ end
+
+ def assert_less_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}")
+ end
+ end
+end
diff --git a/thread.c b/thread.c
index 708aaa471d99..d68a59e9f2d6 100644
--- a/thread.c
+++ b/thread.c
@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval)
return rb_vm_thread_backtrace_locations(argc, argv, thval);
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+rb_thread_t *
+ruby_threadptr_for_trace_memory_allocations(void)
+{
+ // The order of this checks is important due
+ // to how Ruby VM is initialized
+ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
+ return GET_THREAD();
+ }
+
+ return NULL;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations(VALUE _)
+{
+ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
+{
+ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
+ return val;
+}
+
+static VALUE
+rb_thread_memory_allocations(VALUE self)
+{
+ rb_thread_t *th = rb_thread_ptr(self);
+
+ if (!th->vm->thread_trace_memory_allocations) {
+ return Qnil;
+ }
+
+ VALUE ret = rb_hash_new();
+
+ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
+ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
+ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
+
+ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
+ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
+ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
+
+ return ret;
+}
+#endif
+
/*
* Document-class: ThreadError
*
@@ -5230,6 +5279,12 @@ Init_Thread(void)
rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
rb_define_alias(rb_cThread, "inspect", "to_s");
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
+ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
+#endif
+
rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
"stream closed in another thread");
diff --git a/vm_core.h b/vm_core.h
index 12c3ac377551..63cdf55fa6ed 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -69,6 +69,13 @@
# define VM_INSN_INFO_TABLE_IMPL 2
#endif
+/*
+ * track a per thread memory allocations
+ */
+#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
+# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
+#endif
+
#include "ruby/ruby.h"
#include "ruby/st.h"
@@ -602,6 +609,7 @@ typedef struct rb_vm_struct {
unsigned int running: 1;
unsigned int thread_abort_on_exception: 1;
unsigned int thread_report_on_exception: 1;
+ unsigned int thread_trace_memory_allocations: 1;
unsigned int safe_level_: 1;
int sleeper;
@@ -960,6 +968,14 @@ typedef struct rb_thread_struct {
rb_thread_list_t *join_list;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ struct {
+ size_t total_allocated_objects;
+ size_t total_malloc_bytes;
+ size_t total_mallocs;
+ } memory_allocations;
+#endif
+
union {
struct {
VALUE proc;
@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
+rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
void rb_execution_context_update(const rb_execution_context_t *ec);

25
patches/ruby/2.7.7/debug-segfault.patch
View file

@ -1,25 +0,0 @@
diff --git a/class.c b/class.c
index c866d1d727..37ff3c5ade 100644
--- a/class.c
+++ b/class.c
@@ -27,6 +27,7 @@
#include "ruby/st.h"
#include "constant.h"
#include "vm_core.h"
+#include "vm_debug.h"
#include "id_table.h"
#include <ctype.h>
@@ -119,6 +120,12 @@ rb_class_foreach_subclass(VALUE klass, void (*f)(VALUE, VALUE), VALUE arg)
while (cur) {
VALUE curklass = cur->klass;
cur = cur->next;
+
+ if (curklass == 0) {
+ fprintf(stderr, "=== Detected NULL subclass:\n");
+ dp(curklass);
+ }
+
f(curklass, arg);
}
}

258
patches/ruby/2.7.7/thread-memory-allocations-2.7.patch
View file

@ -1,258 +0,0 @@
From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 22 Dec 2020 15:33:08 +0100
Subject: [PATCH] Expose `Thread#memory_allocations` counters
This provides currently a per-thread GC heap slots
and malloc allocations statistics.
This is designed to measure a memory allocations
in a multi-threaded environments (concurrent requests
processing) with an accurate information about allocated
memory within a given execution context.
Example: Measure memory pressure generated by a given
requests to easier find requests with a lot of allocations.
Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530
---
gc.c | 20 ++++++
.../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++
thread.c | 55 +++++++++++++++
vm_core.h | 17 +++++
4 files changed, 159 insertions(+)
create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb
diff --git a/gc.c b/gc.c
index 73faf46b128b..f2dcd2935052 100644
--- a/gc.c
+++ b/gc.c
@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
+ }
+#endif
+
objspace->total_allocated_objects++;
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ if (new_size > old_size) {
+ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
+ }
+
+ if (type == MEMOP_TYPE_MALLOC) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
+ }
+ }
+#endif
+
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) {
diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb
new file mode 100644
index 000000000000..2e281513578b
--- /dev/null
+++ b/test/ruby/test_thread_trace_memory_allocations.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+
+class TestThreadTraceMemoryAllocations < Test::Unit::TestCase
+ def test_disabled_trace_memory_allocations
+ Thread.trace_memory_allocations = false
+
+ assert_predicate Thread.current.memory_allocations, :nil?
+ end
+
+ def test_enabled_trace_memory_allocations
+ Thread.trace_memory_allocations = true
+
+ assert_not_nil(Thread.current.memory_allocations)
+ end
+
+ def test_only_this_thread_allocations_are_counted
+ changed = {
+ total_allocated_objects: 1000,
+ total_malloc_bytes: 1_000_000,
+ total_mallocs: 100
+ }
+
+ Thread.trace_memory_allocations = true
+
+ assert_less_than(changed) do
+ Thread.new do
+ assert_greater_than(changed) do
+ # This will allocate: 5k objects, 5k mallocs, 5MB
+ allocate(5000, 1000)
+ end
+ end.join
+
+ # This will allocate: 50 objects, 50 mallocs, 500 bytes
+ allocate(50, 10)
+ end
+ end
+
+ private
+
+ def allocate(slots, bytes)
+ Array.new(slots).map do
+ '0' * bytes
+ end
+ end
+
+ def assert_greater_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}")
+ end
+ end
+
+ def assert_less_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}")
+ end
+ end
+end
diff --git a/thread.c b/thread.c
index 708aaa471d99..d68a59e9f2d6 100644
--- a/thread.c
+++ b/thread.c
@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval)
return rb_vm_thread_backtrace_locations(argc, argv, thval);
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+rb_thread_t *
+ruby_threadptr_for_trace_memory_allocations(void)
+{
+ // The order of this checks is important due
+ // to how Ruby VM is initialized
+ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
+ return GET_THREAD();
+ }
+
+ return NULL;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations(VALUE _)
+{
+ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
+{
+ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
+ return val;
+}
+
+static VALUE
+rb_thread_memory_allocations(VALUE self)
+{
+ rb_thread_t *th = rb_thread_ptr(self);
+
+ if (!th->vm->thread_trace_memory_allocations) {
+ return Qnil;
+ }
+
+ VALUE ret = rb_hash_new();
+
+ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
+ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
+ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
+
+ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
+ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
+ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
+
+ return ret;
+}
+#endif
+
/*
* Document-class: ThreadError
*
@@ -5230,6 +5279,12 @@ Init_Thread(void)
rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
rb_define_alias(rb_cThread, "inspect", "to_s");
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
+ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
+#endif
+
rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
"stream closed in another thread");
diff --git a/vm_core.h b/vm_core.h
index 12c3ac377551..63cdf55fa6ed 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -69,6 +69,13 @@
# define VM_INSN_INFO_TABLE_IMPL 2
#endif
+/*
+ * track a per thread memory allocations
+ */
+#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
+# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
+#endif
+
#include "ruby/ruby.h"
#include "ruby/st.h"
@@ -602,6 +609,7 @@ typedef struct rb_vm_struct {
unsigned int running: 1;
unsigned int thread_abort_on_exception: 1;
unsigned int thread_report_on_exception: 1;
+ unsigned int thread_trace_memory_allocations: 1;
unsigned int safe_level_: 1;
int sleeper;
@@ -960,6 +968,14 @@ typedef struct rb_thread_struct {
rb_thread_list_t *join_list;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ struct {
+ size_t total_allocated_objects;
+ size_t total_malloc_bytes;
+ size_t total_mallocs;
+ } memory_allocations;
+#endif
+
union {
struct {
VALUE proc;
@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
+rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
void rb_execution_context_update(const rb_execution_context_t *ec);

0
patches/ruby/2.7.2/debug-segfault.patch → patches/ruby/2.7/debug-segfault.patch
View file

0
patches/ruby/2.7.2/thread-memory-allocations-2.7.patch → patches/ruby/2.7/thread-memory-allocations-2.7.patch
View file

22
patches/ruby/3.0.3/dont_alias_array_size.patch
View file

@ -1,22 +0,0 @@
commit fa0279d947c3962c3f8c32852278d3ebb964cb19
Author: Koichi Sasada <ko1@atdot.net>
Date: Wed Jul 28 13:40:30 2021 +0900
should not share same `def` for specialized method
Because the key of redefine table is `def`, `def` should be
unique for each optimized method (`alias` is not allowed).
diff --git a/array.c b/array.c
index 3cb57a0872..36f712bcac 100644
--- a/array.c
+++ b/array.c
@@ -8384,7 +8384,7 @@ Init_Array(void)
rb_define_method(rb_cArray, "each_index", rb_ary_each_index, 0);
rb_define_method(rb_cArray, "reverse_each", rb_ary_reverse_each, 0);
rb_define_method(rb_cArray, "length", rb_ary_length, 0);
- rb_define_alias(rb_cArray, "size", "length");
+ rb_define_method(rb_cArray, "size", rb_ary_length, 0);
rb_define_method(rb_cArray, "empty?", rb_ary_empty_p, 0);
rb_define_method(rb_cArray, "find_index", rb_ary_index, -1);
rb_define_method(rb_cArray, "index", rb_ary_index, -1);

85
patches/ruby/3.0.3/override_optimized_methods.patch
View file

@ -1,85 +0,0 @@
commit fb4cf204a662a8cd9dafef6f31f2bd0db9129abe
Author: Koichi Sasada <ko1@atdot.net>
Date: Thu May 13 03:10:18 2021 +0900
use me->def instead of me for opt_table
`vm_opt_method_table` is me=>bop table to manage the optimized
methods (by specialized instruction). However, `me` can be invalidated
to invalidate the method cache entry.
[Bug #17725]
To solve the issue, use `me-def` instead of `me` which simply copied
at invalidation timing.
A test by @jeremyevans https://github.com/ruby/ruby/pull/4376
diff --git a/test/ruby/test_method.rb b/test/ruby/test_method.rb
index 240821c9e2..0bd5dc63dd 100644
--- a/test/ruby/test_method.rb
+++ b/test/ruby/test_method.rb
@@ -1303,6 +1303,21 @@ class TestMethod < Test::Unit::TestCase
end;
end
+ def test_override_optimized_method_on_class_using_prepend
+ assert_separately(%w(--disable-gems), <<-'end;', timeout: 30)
+ # Bug #17725 [ruby-core:102884]
+ $VERBOSE = nil
+ String.prepend(Module.new)
+ class String
+ def + other
+ 'blah blah'
+ end
+ end
+
+ assert_equal('blah blah', 'a' + 'b')
+ end;
+ end
+
def test_eqq
assert_operator(0.method(:<), :===, 5)
assert_not_operator(0.method(:<), :===, -5)
diff --git a/vm.c b/vm.c
index accd12644e..8a044efa1f 100644
--- a/vm.c
+++ b/vm.c
@@ -1798,7 +1798,7 @@ rb_iter_break_value(VALUE val)
/* optimization: redefine management */
-static st_table *vm_opt_method_table = 0;
+static st_table *vm_opt_method_def_table = 0;
static st_table *vm_opt_mid_table = 0;
static int
@@ -1852,9 +1852,8 @@ rb_vm_check_redefinition_opt_method(const rb_method_entry_t *me, VALUE klass)
klass = RBASIC_CLASS(klass);
}
if (vm_redefinition_check_method_type(me->def)) {
- if (st_lookup(vm_opt_method_table, (st_data_t)me, &bop)) {
- int flag = vm_redefinition_check_flag(klass);
-
+ if (st_lookup(vm_opt_method_def_table, (st_data_t)me->def, &bop)) {
+ int flag = vm_redefinition_check_flag(klass);
ruby_vm_redefined_flag[bop] |= flag;
}
}
@@ -1885,7 +1884,7 @@ add_opt_method(VALUE klass, ID mid, VALUE bop)
const rb_method_entry_t *me = rb_method_entry_at(klass, mid);
if (me && vm_redefinition_check_method_type(me->def)) {
- st_insert(vm_opt_method_table, (st_data_t)me, (st_data_t)bop);
+ st_insert(vm_opt_method_def_table, (st_data_t)me->def, (st_data_t)bop);
st_insert(vm_opt_mid_table, (st_data_t)mid, (st_data_t)Qtrue);
}
else {
@@ -1899,7 +1898,7 @@ vm_init_redefined_flag(void)
ID mid;
VALUE bop;
- vm_opt_method_table = st_init_numtable();
+ vm_opt_method_def_table = st_init_numtable();
vm_opt_mid_table = st_init_numtable();
#define OP(mid_, bop_) (mid = id##mid_, bop = BOP_##bop_, ruby_vm_redefined_flag[bop] = 0)

258
patches/ruby/3.0.4/thread-memory-allocations-3.0.patch
View file

@ -1,258 +0,0 @@
From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Tue, 22 Dec 2020 15:33:08 +0100
Subject: [PATCH] Expose `Thread#memory_allocations` counters
This provides currently a per-thread GC heap slots
and malloc allocations statistics.
This is designed to measure a memory allocations
in a multi-threaded environments (concurrent requests
processing) with an accurate information about allocated
memory within a given execution context.
Example: Measure memory pressure generated by a given
requests to easier find requests with a lot of allocations.
Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530
---
gc.c | 20 ++++++
.../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++
thread.c | 55 +++++++++++++++
vm_core.h | 17 +++++
4 files changed, 159 insertions(+)
create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb
diff --git a/gc.c b/gc.c
index 27cf65b196a3..280c62fbe341 100644
--- a/gc.c
+++ b/gc.c
@@ -2123,6 +2123,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
// TODO: make it atomic, or ractor local
objspace->total_allocated_objects++;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
+ }
+#endif
+
#if RGENGC_PROFILE
if (wb_protected) {
objspace->profile.total_generated_normal_object_count++;
@@ -10487,6 +10494,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si
#endif
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
+ if (th) {
+ if (new_size > old_size) {
+ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
+ }
+
+ if (type == MEMOP_TYPE_MALLOC) {
+ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
+ }
+ }
+#endif
+
if (type == MEMOP_TYPE_MALLOC) {
retry:
if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb
new file mode 100644
index 000000000000..2e281513578b
--- /dev/null
+++ b/test/ruby/test_thread_trace_memory_allocations.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'test/unit'
+
+class TestThreadTraceMemoryAllocations < Test::Unit::TestCase
+ def test_disabled_trace_memory_allocations
+ Thread.trace_memory_allocations = false
+
+ assert_predicate Thread.current.memory_allocations, :nil?
+ end
+
+ def test_enabled_trace_memory_allocations
+ Thread.trace_memory_allocations = true
+
+ assert_not_nil(Thread.current.memory_allocations)
+ end
+
+ def test_only_this_thread_allocations_are_counted
+ changed = {
+ total_allocated_objects: 1000,
+ total_malloc_bytes: 1_000_000,
+ total_mallocs: 100
+ }
+
+ Thread.trace_memory_allocations = true
+
+ assert_less_than(changed) do
+ Thread.new do
+ assert_greater_than(changed) do
+ # This will allocate: 5k objects, 5k mallocs, 5MB
+ allocate(5000, 1000)
+ end
+ end.join
+
+ # This will allocate: 50 objects, 50 mallocs, 500 bytes
+ allocate(50, 10)
+ end
+ end
+
+ private
+
+ def allocate(slots, bytes)
+ Array.new(slots).map do
+ '0' * bytes
+ end
+ end
+
+ def assert_greater_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}")
+ end
+ end
+
+ def assert_less_than(keys)
+ before = Thread.current.memory_allocations
+ yield
+ after = Thread.current.memory_allocations
+
+ keys.each do |key, by|
+ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}")
+ end
+ end
+end
diff --git a/thread.c b/thread.c
index dce181d24e02..247440766cdf 100644
--- a/thread.c
+++ b/thread.c
@@ -5412,6 +5412,55 @@ Init_Thread_Mutex(void)
rb_native_mutex_initialize(&th->interrupt_lock);
}
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+rb_thread_t *
+ruby_threadptr_for_trace_memory_allocations(void)
+{
+ // The order of this checks is important due
+ // to how Ruby VM is initialized
+ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
+ return GET_THREAD();
+ }
+
+ return NULL;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations(VALUE _)
+{
+ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
+}
+
+static VALUE
+rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
+{
+ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
+ return val;
+}
+
+static VALUE
+rb_thread_memory_allocations(VALUE self)
+{
+ rb_thread_t *th = rb_thread_ptr(self);
+
+ if (!th->vm->thread_trace_memory_allocations) {
+ return Qnil;
+ }
+
+ VALUE ret = rb_hash_new();
+
+ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
+ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
+ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
+
+ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
+ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
+ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
+
+ return ret;
+}
+#endif
+
/*
* Document-class: ThreadError
*
@@ -5497,6 +5546,12 @@ Init_Thread(void)
rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
rb_define_alias(rb_cThread, "inspect", "to_s");
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
+ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
+ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
+#endif
+
rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
"stream closed in another thread");
diff --git a/vm_core.h b/vm_core.h
index 5f8d4ab87670..ac15f72fa25b 100644
--- a/vm_core.h
+++ b/vm_core.h
@@ -97,6 +97,13 @@
# define VM_INSN_INFO_TABLE_IMPL 2
#endif
+/*
+ * track a per thread memory allocations
+ */
+#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
+# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
+#endif
+
#if defined(NSIG_MAX) /* POSIX issue 8 */
# undef NSIG
# define NSIG NSIG_MAX
@@ -606,6 +613,7 @@ typedef struct rb_vm_struct {
unsigned int thread_abort_on_exception: 1;
unsigned int thread_report_on_exception: 1;
unsigned int thread_ignore_deadlock: 1;
+ unsigned int thread_trace_memory_allocations: 1;
/* object management */
VALUE mark_object_ary;
@@ -981,6 +989,14 @@ typedef struct rb_thread_struct {
struct rb_waiting_list *join_list;
+#if THREAD_TRACE_MEMORY_ALLOCATIONS
+ struct {
+ size_t total_allocated_objects;
+ size_t total_malloc_bytes;
+ size_t total_mallocs;
+ } memory_allocations;
+#endif
+
union {
struct {
VALUE proc;
@@ -1901,6 +1917,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
+rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
void rb_execution_context_update(const rb_execution_context_t *ec);

0
patches/ruby/3.0.3/thread-memory-allocations-3.0.patch → patches/ruby/3.0/thread-memory-allocations-3.0.patch
View file

5
scripts/install-ruby
View file

@ -47,8 +47,9 @@ function build_common() {
cd /usr/src/ruby
# Apply patches
if [[ -d "/patches/ruby/$RUBY_VERSION" ]]; then
for i in "/patches/ruby/$RUBY_VERSION"/*.patch; do
ruby_version=`cut -d '.' -f 1,2 <<< $RUBY_VERSION`
if [[ -d "/patches/ruby/$ruby_version" ]]; then
for i in "/patches/ruby/$ruby_version"/*.patch; do
echo "$i..."
patch -p1 -i "$i"
done