From a8f2113955a83d2defba031688cf853897fcc083 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 8 Mar 2023 13:05:05 -0800 Subject: [PATCH 1/3] Add Ruby 3.1 to build Ruby 3.2 contains a number of kwargs fixes, and Rails 7 may be a prerequisite for Ruby 3.2. We'll want to test and deploy Ruby 3.1 before jumping to 3.2. Ruby 3.1 requires at least rubygems v3.1.3. Let's just bump everything to 3.4, and consolidate the Ruby 3.2 build. --- .gitlab/ci/gitlab.images.yml | 32 +--- .../ruby/3.1/thread-memory-allocations.patch | 160 ++++++++++++++++++ scripts/lib/custom-docker-build | 5 + 3 files changed, 171 insertions(+), 26 deletions(-) create mode 100644 patches/ruby/3.1/thread-memory-allocations.patch diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index 3041ef0..f426f23 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -13,26 +13,6 @@ gitlab: extends: - .gitlab stage: gitlab - variables: - RUBYGEMS: '3.2' - LFS: '2.9' - YARN: '1.22' - GRAPHICSMAGICK: '1.3.36' - ARCH: linux/amd64,linux/arm64 - parallel: - matrix: - - OS: ['debian:bullseye'] - RUBY: ['2.7.patched', '3.0.patched'] - GIT: ['2.36'] - POSTGRESQL: ['11', '12', '13'] - GOLANG: ['1.18', '1.19'] - RUST: ['1.65'] - NODE: ['16.14', '18.12'] - CHROME: ['106', '109'] - -gitlab-ruby-3.2: - extends: - - .gitlab variables: RUBYGEMS: '3.4' LFS: '2.9' @@ -42,13 +22,13 @@ gitlab-ruby-3.2: parallel: matrix: - OS: ['debian:bullseye'] - RUBY: ['3.2.patched'] + RUBY: ['2.7.patched', '3.0.patched', '3.1.patched', '3.2.patched'] GIT: ['2.36'] POSTGRESQL: ['11', '12', '13'] - GOLANG: ['1.19'] + GOLANG: ['1.18', '1.19'] RUST: ['1.65'] - NODE: ['16.14'] - CHROME: ['109'] + NODE: ['16.14', '18.12'] + CHROME: ['106', '109'] # Used by GitLab's compile-production-assets and compile-test-assets jobs gitlab-assets: @@ -56,7 +36,7 @@ gitlab-assets: - .gitlab stage: gitlab-assets variables: - RUBYGEMS: '3.2' + RUBYGEMS: '3.4' LFS: '2.9' YARN: '1.22' GRAPHICSMAGICK: '1.3.36' @@ -64,7 +44,7 @@ gitlab-assets: parallel: matrix: - OS: ['debian:bullseye'] - RUBY: ['2.7', '3.0'] + RUBY: ['2.7', '3.0', '3.1', '3.2'] GIT: ['2.33'] NODE: ['16.14', '18.12'] diff --git a/patches/ruby/3.1/thread-memory-allocations.patch b/patches/ruby/3.1/thread-memory-allocations.patch new file mode 100644 index 0000000..876f9e2 --- /dev/null +++ b/patches/ruby/3.1/thread-memory-allocations.patch @@ -0,0 +1,160 @@ +diff --git a/gc.c b/gc.c +index 030a4627bd..5e57e4bbfd 100644 +--- a/gc.c ++++ b/gc.c +@@ -2285,6 +2285,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, + // TODO: make it atomic, or ractor local + objspace->total_allocated_objects++; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + #if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +@@ -11305,6 +11312,19 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +diff --git a/thread.c b/thread.c +index de29340713..c02f2b7945 100644 +--- a/thread.c ++++ b/thread.c +@@ -5402,6 +5402,55 @@ Init_Thread_Mutex(void) + rb_native_mutex_initialize(&th->interrupt_lock); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5488,6 +5537,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 11866b85e5..f36b582cc7 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -94,6 +94,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #if defined(NSIG_MAX) /* POSIX issue 8 */ + # undef NSIG + # define NSIG NSIG_MAX +@@ -663,6 +670,7 @@ typedef struct rb_vm_struct { + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; + unsigned int thread_ignore_deadlock: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + /* object management */ + VALUE mark_object_ary; +@@ -1048,6 +1056,14 @@ typedef struct rb_thread_struct { + + struct rb_waiting_list *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1973,6 +1989,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index 4bc17ac..ae8dd66 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -239,6 +239,11 @@ function print_ruby_args() { RUBY_DOWNLOAD_SHA256="9afc6380a027a4fe1ae1a3e2eccb6b497b9c5ac0631c12ca56f9b7beb4848776" ;; + 3.1|3.1.patched) + RUBY_VERSION="3.1.3" + RUBY_DOWNLOAD_SHA256="5ea498a35f4cd15875200a52dde42b6eb179e1264e17d78732c3a57cd1c6ab9e" + ;; + 3.2|3.2.patched) RUBY_VERSION="3.2.0" RUBY_DOWNLOAD_SHA256="daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" From 7cd47b00a8c24a8374c5a9d00d8c8b0bb4ed5075 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 8 Mar 2023 13:24:17 -0800 Subject: [PATCH 2/3] Bump Ruby 3.2 version to 3.2.1 --- scripts/lib/custom-docker-build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index ae8dd66..c807f45 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -245,8 +245,8 @@ function print_ruby_args() { ;; 3.2|3.2.patched) - RUBY_VERSION="3.2.0" - RUBY_DOWNLOAD_SHA256="daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" + RUBY_VERSION="3.2.1" + RUBY_DOWNLOAD_SHA256="13d67901660ee3217dbd9dd56059346bd4212ce64a69c306ef52df64935f8dbd" ;; From fb6c042b8a0dbd0a2b55e74cb89a55a2df6b20d6 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 8 Mar 2023 14:16:24 -0800 Subject: [PATCH 3/3] Drop Chrome 106 from the builds Chrome 109 appears to be the default, so let's stick with that. --- .gitlab/ci/gitlab.images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index f426f23..b10c17f 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -28,7 +28,7 @@ gitlab: GOLANG: ['1.18', '1.19'] RUST: ['1.65'] NODE: ['16.14', '18.12'] - CHROME: ['106', '109'] + CHROME: ['109'] # Used by GitLab's compile-production-assets and compile-test-assets jobs gitlab-assets: