From bc3223ac21ee1c91b1fb609f1d574244ad371257 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 18 Jan 2023 13:24:58 -0800 Subject: [PATCH] Add Ruby 3.2.0 images This will be used to see how much effort it would be to jump to Ruby 3.2.0 and to see what performance improvements we might see with YJIT. --- .gitlab/ci/gitlab.images.yml | 21 +++ .../ruby/3.2/thread-memory-allocations.patch | 160 ++++++++++++++++++ scripts/lib/custom-docker-build | 9 + 3 files changed, 190 insertions(+) create mode 100644 patches/ruby/3.2/thread-memory-allocations.patch diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index 7454fa1..14b8a66 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -20,6 +20,27 @@ gitlab: NODE: ['16.14', '18.12'] CHROME: ['106', '108'] +gitlab-ruby-3.2: + extends: + - .build_and_push + stage: gitlab + variables: + RUBYGEMS: '3.4' + LFS: '2.9' + YARN: '1.22' + GRAPHICSMAGICK: '1.3.36' + ARCH: linux/amd64,linux/arm64 + parallel: + matrix: + - OS: ['debian:bullseye'] + RUBY: ['3.2.patched'] + GIT: ['2.36'] + POSTGRESQL: ['11', '12', '13'] + GOLANG: ['1.19'] + RUST: ['1.65.0'] + NODE: ['16.14'] + CHROME: ['108'] + # Used by GitLab's compile-production-assets and compile-test-assets jobs gitlab-assets: extends: diff --git a/patches/ruby/3.2/thread-memory-allocations.patch b/patches/ruby/3.2/thread-memory-allocations.patch new file mode 100644 index 0000000..737dfdb --- /dev/null +++ b/patches/ruby/3.2/thread-memory-allocations.patch @@ -0,0 +1,160 @@ +diff --git a/gc.c b/gc.c +index 0fccc17a32..49a4cc21b6 100644 +--- a/gc.c ++++ b/gc.c +@@ -2541,6 +2541,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, + // TODO: make it atomic, or ractor local + objspace->total_allocated_objects++; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + #if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +@@ -12127,6 +12134,19 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +diff --git a/thread.c b/thread.c +index 56a5644552..176509e1be 100644 +--- a/thread.c ++++ b/thread.c +@@ -5289,6 +5289,55 @@ Init_Thread_Mutex(void) + rb_native_mutex_initialize(&th->interrupt_lock); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5371,6 +5420,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 4f6e07d818..893b578cc8 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -114,6 +114,13 @@ extern int ruby_assert_critical_section_entered; + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #if defined(NSIG_MAX) /* POSIX issue 8 */ + # undef NSIG + # define NSIG NSIG_MAX +@@ -661,6 +668,7 @@ typedef struct rb_vm_struct { + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; + unsigned int thread_ignore_deadlock: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + /* object management */ + VALUE mark_object_ary; +@@ -1040,6 +1048,14 @@ typedef struct rb_thread_struct { + + struct rb_waiting_list *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1967,6 +1983,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(rb_execution_context_t *ec); diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index edabcbd..5772e3c 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -249,6 +249,12 @@ function print_ruby_args() { RUBY_DOWNLOAD_SHA256="9afc6380a027a4fe1ae1a3e2eccb6b497b9c5ac0631c12ca56f9b7beb4848776" ;; + 3.2|3.2.patched) + RUBY_VERSION="3.2.0" + RUBY_DOWNLOAD_SHA256="daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" + ;; + + *) fail "Unknown ruby version $1" ;; esac @@ -278,6 +284,9 @@ function print_rubygems_args() { 3.2) RUBYGEMS_VERSION=3.2.33 ;; + 3.4) + RUBYGEMS_VERSION=3.4.4 + ;; *) fail "Unknown rubygems version $1" ;; esac