From d25cb1006751eb424ef7ad77938633a74eccf16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 22 Jan 2021 12:02:46 +0100 Subject: [PATCH 1/5] Apply thread allocations patch to Ruby 2.7 This makes us to compile Ruby 2.7 manually and apply relevant patch. --- .gitlab-ci.yml | 1 + Dockerfile.custom | 6 +++++ scripts/custom-docker-build | 45 +++++++++++++++++++------------- scripts/install-ruby | 52 +++++++++++++++++++++++++++++++------ 4 files changed, 78 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7b3eba..abcaf25 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ default: before_script: - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" "$CI_REGISTRY" - source scripts/build-helpers.sh + - apk add -U bash tags: - gitlab-org-docker diff --git a/Dockerfile.custom b/Dockerfile.custom index c44c031..2b6c8bf 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -13,6 +13,12 @@ RUN /scripts/install-essentials ENV PATH $PATH:/usr/local/go/bin +# Ruby +ARG RUBY_VERSION +ARG RUBY_DOWNLOAD_SHA256 + +RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 && ruby --version; fi + # Git ARG GIT_VERSION ARG GIT_DOWNLOAD_URL diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 6ff73bd..02ca47f 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e IFS=$'\n\t' @@ -228,29 +228,38 @@ function print_pgbouncer_args() { printf -- "--build-arg PGBOUNCER_DOWNLOAD_SHA256=%s " "$PGBOUNCER_DOWNLOAD_SHA256" } -function parse_arguments() { - read base - read base_version +function print_ruby_args() { + case "$1" in + 2.6.*) + CUSTOM_IMAGE_NAME=debian + CUSTOM_IMAGE_VERSION=stretch + RUBY_VERSION="2.6.6" + RUBY_DOWNLOAD_SHA256="5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f" + ;; - # Lock Ruby to Debian version to pin OpenSSL version - case "$base" in - ruby) - case "$base_version" in - 2.6.*) - base_version="$base_version-stretch" - ;; - *) - base_version="$base_version-buster" - ;; - esac + 2.7.*) + CUSTOM_IMAGE_NAME=debian + CUSTOM_IMAGE_VERSION=buster + RUBY_VERSION="2.7.2" + RUBY_DOWNLOAD_SHA256="1b95ab193cc8f5b5e59d2686cb3d5dcf1ddf2a86cb6950e0b4bdaae5040ec0d6" + ;; + + *) echo "Unknown ruby version $1"; exit 1; esac - printf -- "-f Dockerfile.custom " "$base" - printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$base" - printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$base_version" + printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" + printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION" + printf -- "--build-arg RUBY_VERSION=%s " "$RUBY_VERSION" + printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256" +} + +function parse_arguments() { + printf -- "-f Dockerfile.custom " + while read image; do read version case "$image" in + ruby) print_ruby_args $version ;; golang) print_golang_args $version ;; chrome) print_chrome_args $version ;; docker) print_docker_args $version ;; diff --git a/scripts/install-ruby b/scripts/install-ruby index 7aa1212..5a8a3d6 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -1,14 +1,50 @@ #!/bin/bash -set -xeuo pipefail -IFS=$'\n\t' -cd /tmp -wget http://cache.ruby-lang.org/pub/ruby/2.2/ruby-2.2.2.tar.gz -tar -xvzf ruby-2.2.2.tar.gz +set -xeou pipefail -cd ruby-2.2.2 +# Based on https://github.com/docker-library/ruby/blob/master/2.7/buster/Dockerfile + +RUBY_VERSION=${1} +RUBY_MAJOR=${1%.*} # strip last component +RUBY_DOWNLOAD_SHA256=${2} + +RUBY_DOWNLOAD_URL="https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" + +# Download Ruby +curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.xz +echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.xz" | sha256sum -c - + +# Skip installing Gem docs +mkdir -p /usr/local/etc +echo 'install: --no-document' >> /usr/local/etc/gemrc +echo 'update: --no-document' >> /usr/local/etc/gemrc + +# Install needed packages +apt-get update +apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev ruby + +# Unpack Ruby +mkdir -p /usr/src/ruby +tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 +rm ruby.tar.xz +cd /usr/src/ruby + +# Apply patches +find /patches/ruby/$RUBY_VERSION -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -i ./configure --enable-shared --disable-install-doc --disable-install-rdoc --disable-install-capi -make install -j 3 +make install -j $(nproc) +# Cleanup cd / -rm -rf /tmp/* +rm -rf /usr/src/ruby +apt-get purge -y --auto-remove ruby + +# Verify +# verify we have no "ruby" packages installed +! dpkg -l | grep -i ruby +[ "$(command -v ruby)" = '/usr/local/bin/ruby' ] + +# rough smoke test +ruby --version +gem --version +bundle --version From 0b0a3b745aca397bc67998246e1d03a41bc8708c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 1 Feb 2021 14:20:08 +0100 Subject: [PATCH 2/5] Add missing patch --- .../2.7/thread-memory-allocations-2.7.patch | 258 ++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 patches/ruby/2.7/thread-memory-allocations-2.7.patch diff --git a/patches/ruby/2.7/thread-memory-allocations-2.7.patch b/patches/ruby/2.7/thread-memory-allocations-2.7.patch new file mode 100644 index 0000000..ca0a87e --- /dev/null +++ b/patches/ruby/2.7/thread-memory-allocations-2.7.patch @@ -0,0 +1,258 @@ +From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= +Date: Tue, 22 Dec 2020 15:33:08 +0100 +Subject: [PATCH] Expose `Thread#memory_allocations` counters + +This provides currently a per-thread GC heap slots +and malloc allocations statistics. + +This is designed to measure a memory allocations +in a multi-threaded environments (concurrent requests +processing) with an accurate information about allocated +memory within a given execution context. + +Example: Measure memory pressure generated by a given +requests to easier find requests with a lot of allocations. + +Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530 +--- + gc.c | 20 ++++++ + .../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++ + thread.c | 55 +++++++++++++++ + vm_core.h | 17 +++++ + 4 files changed, 159 insertions(+) + create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb + +diff --git a/gc.c b/gc.c +index 73faf46b128b..f2dcd2935052 100644 +--- a/gc.c ++++ b/gc.c +@@ -2172,6 +2172,13 @@ newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_prote + GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */ + #endif + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + objspace->total_allocated_objects++; + + gc_report(5, objspace, "newobj: %s\n", obj_info(obj)); +@@ -9732,6 +9739,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc) { +diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb +new file mode 100644 +index 000000000000..2e281513578b +--- /dev/null ++++ b/test/ruby/test_thread_trace_memory_allocations.rb +@@ -0,0 +1,67 @@ ++# frozen_string_literal: true ++ ++require 'test/unit' ++ ++class TestThreadTraceMemoryAllocations < Test::Unit::TestCase ++ def test_disabled_trace_memory_allocations ++ Thread.trace_memory_allocations = false ++ ++ assert_predicate Thread.current.memory_allocations, :nil? ++ end ++ ++ def test_enabled_trace_memory_allocations ++ Thread.trace_memory_allocations = true ++ ++ assert_not_nil(Thread.current.memory_allocations) ++ end ++ ++ def test_only_this_thread_allocations_are_counted ++ changed = { ++ total_allocated_objects: 1000, ++ total_malloc_bytes: 1_000_000, ++ total_mallocs: 100 ++ } ++ ++ Thread.trace_memory_allocations = true ++ ++ assert_less_than(changed) do ++ Thread.new do ++ assert_greater_than(changed) do ++ # This will allocate: 5k objects, 5k mallocs, 5MB ++ allocate(5000, 1000) ++ end ++ end.join ++ ++ # This will allocate: 50 objects, 50 mallocs, 500 bytes ++ allocate(50, 10) ++ end ++ end ++ ++ private ++ ++ def allocate(slots, bytes) ++ Array.new(slots).map do ++ '0' * bytes ++ end ++ end ++ ++ def assert_greater_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}") ++ end ++ end ++ ++ def assert_less_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}") ++ end ++ end ++end +diff --git a/thread.c b/thread.c +index 708aaa471d99..d68a59e9f2d6 100644 +--- a/thread.c ++++ b/thread.c +@@ -5143,6 +5143,55 @@ rb_thread_backtrace_locations_m(int argc, VALUE *argv, VALUE thval) + return rb_vm_thread_backtrace_locations(argc, argv, thval); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5230,6 +5279,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 12c3ac377551..63cdf55fa6ed 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -69,6 +69,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #include "ruby/ruby.h" + #include "ruby/st.h" + +@@ -602,6 +609,7 @@ typedef struct rb_vm_struct { + unsigned int running: 1; + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + unsigned int safe_level_: 1; + int sleeper; +@@ -960,6 +968,14 @@ typedef struct rb_thread_struct { + + rb_thread_list_t *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1852,6 +1868,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); From f0bf9d8f0c39ca3370b0a53c4fee1e23c0ef9fd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 1 Feb 2021 14:20:08 +0100 Subject: [PATCH 3/5] Fix support for Ruby versions --- .../thread-memory-allocations-2.7.patch | 0 .../3.0.0/thread-memory-allocations-3.0.patch | 258 ++++++++++++++++++ scripts/custom-docker-build | 15 +- scripts/install-ruby | 10 +- 4 files changed, 274 insertions(+), 9 deletions(-) rename patches/ruby/{2.7 => 2.7.2}/thread-memory-allocations-2.7.patch (100%) create mode 100644 patches/ruby/3.0.0/thread-memory-allocations-3.0.patch diff --git a/patches/ruby/2.7/thread-memory-allocations-2.7.patch b/patches/ruby/2.7.2/thread-memory-allocations-2.7.patch similarity index 100% rename from patches/ruby/2.7/thread-memory-allocations-2.7.patch rename to patches/ruby/2.7.2/thread-memory-allocations-2.7.patch diff --git a/patches/ruby/3.0.0/thread-memory-allocations-3.0.patch b/patches/ruby/3.0.0/thread-memory-allocations-3.0.patch new file mode 100644 index 0000000..ba3a743 --- /dev/null +++ b/patches/ruby/3.0.0/thread-memory-allocations-3.0.patch @@ -0,0 +1,258 @@ +From 97f14ebfd8d24d71e10c450e0a90b6322f9c0d59 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= +Date: Tue, 22 Dec 2020 15:33:08 +0100 +Subject: [PATCH] Expose `Thread#memory_allocations` counters + +This provides currently a per-thread GC heap slots +and malloc allocations statistics. + +This is designed to measure a memory allocations +in a multi-threaded environments (concurrent requests +processing) with an accurate information about allocated +memory within a given execution context. + +Example: Measure memory pressure generated by a given +requests to easier find requests with a lot of allocations. + +Ref: https://gitlab.com/gitlab-org/gitlab/-/issues/296530 +--- + gc.c | 20 ++++++ + .../test_thread_trace_memory_allocations.rb | 67 +++++++++++++++++++ + thread.c | 55 +++++++++++++++ + vm_core.h | 17 +++++ + 4 files changed, 159 insertions(+) + create mode 100644 test/ruby/test_thread_trace_memory_allocations.rb + +diff --git a/gc.c b/gc.c +index 27cf65b196a3..280c62fbe341 100644 +--- a/gc.c ++++ b/gc.c +@@ -2123,6 +2123,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, + // TODO: make it atomic, or ractor local + objspace->total_allocated_objects++; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + #if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +@@ -10487,6 +10494,19 @@ objspace_malloc_increase(rb_objspace_t *objspace, void *mem, size_t new_size, si + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +diff --git a/test/ruby/test_thread_trace_memory_allocations.rb b/test/ruby/test_thread_trace_memory_allocations.rb +new file mode 100644 +index 000000000000..2e281513578b +--- /dev/null ++++ b/test/ruby/test_thread_trace_memory_allocations.rb +@@ -0,0 +1,67 @@ ++# frozen_string_literal: true ++ ++require 'test/unit' ++ ++class TestThreadTraceMemoryAllocations < Test::Unit::TestCase ++ def test_disabled_trace_memory_allocations ++ Thread.trace_memory_allocations = false ++ ++ assert_predicate Thread.current.memory_allocations, :nil? ++ end ++ ++ def test_enabled_trace_memory_allocations ++ Thread.trace_memory_allocations = true ++ ++ assert_not_nil(Thread.current.memory_allocations) ++ end ++ ++ def test_only_this_thread_allocations_are_counted ++ changed = { ++ total_allocated_objects: 1000, ++ total_malloc_bytes: 1_000_000, ++ total_mallocs: 100 ++ } ++ ++ Thread.trace_memory_allocations = true ++ ++ assert_less_than(changed) do ++ Thread.new do ++ assert_greater_than(changed) do ++ # This will allocate: 5k objects, 5k mallocs, 5MB ++ allocate(5000, 1000) ++ end ++ end.join ++ ++ # This will allocate: 50 objects, 50 mallocs, 500 bytes ++ allocate(50, 10) ++ end ++ end ++ ++ private ++ ++ def allocate(slots, bytes) ++ Array.new(slots).map do ++ '0' * bytes ++ end ++ end ++ ++ def assert_greater_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :<=, after[key]-before[key], "expected the #{key} to change more than #{by}") ++ end ++ end ++ ++ def assert_less_than(keys) ++ before = Thread.current.memory_allocations ++ yield ++ after = Thread.current.memory_allocations ++ ++ keys.each do |key, by| ++ assert_operator(by, :>, after[key]-before[key], "expected the #{key} to change less than #{by}") ++ end ++ end ++end +diff --git a/thread.c b/thread.c +index dce181d24e02..247440766cdf 100644 +--- a/thread.c ++++ b/thread.c +@@ -5412,6 +5412,55 @@ Init_Thread_Mutex(void) + rb_native_mutex_initialize(&th->interrupt_lock); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5497,6 +5546,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 5f8d4ab87670..ac15f72fa25b 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -97,6 +97,13 @@ + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #if defined(NSIG_MAX) /* POSIX issue 8 */ + # undef NSIG + # define NSIG NSIG_MAX +@@ -606,6 +613,7 @@ typedef struct rb_vm_struct { + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; + unsigned int thread_ignore_deadlock: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + /* object management */ + VALUE mark_object_ary; +@@ -981,6 +989,14 @@ typedef struct rb_thread_struct { + + struct rb_waiting_list *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1901,6 +1917,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(const rb_execution_context_t *ec); diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 02ca47f..ba3dbf8 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -230,18 +230,25 @@ function print_pgbouncer_args() { function print_ruby_args() { case "$1" in - 2.6.*) + 2.6|2.6.*) CUSTOM_IMAGE_NAME=debian CUSTOM_IMAGE_VERSION=stretch RUBY_VERSION="2.6.6" - RUBY_DOWNLOAD_SHA256="5db187882b7ac34016cd48d7032e197f07e4968f406b0690e20193b9b424841f" + RUBY_DOWNLOAD_SHA256="364b143def360bac1b74eb56ed60b1a0dca6439b00157ae11ff77d5cd2e92291" ;; - 2.7.*) + 2.7|2.7.*) CUSTOM_IMAGE_NAME=debian CUSTOM_IMAGE_VERSION=buster RUBY_VERSION="2.7.2" - RUBY_DOWNLOAD_SHA256="1b95ab193cc8f5b5e59d2686cb3d5dcf1ddf2a86cb6950e0b4bdaae5040ec0d6" + RUBY_DOWNLOAD_SHA256="6e5706d0d4ee4e1e2f883db9d768586b4d06567debea353c796ec45e8321c3d4" + ;; + + 3.0|3.0.*) + CUSTOM_IMAGE_NAME=debian + CUSTOM_IMAGE_VERSION=buster + RUBY_VERSION="3.0.0" + RUBY_DOWNLOAD_SHA256="a13ed141a1c18eb967aac1e33f4d6ad5f21be1ac543c344e0d6feeee54af8e28" ;; *) echo "Unknown ruby version $1"; exit 1; diff --git a/scripts/install-ruby b/scripts/install-ruby index 5a8a3d6..90fd24d 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -8,11 +8,11 @@ RUBY_VERSION=${1} RUBY_MAJOR=${1%.*} # strip last component RUBY_DOWNLOAD_SHA256=${2} -RUBY_DOWNLOAD_URL="https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.xz" +RUBY_DOWNLOAD_URL="https://cache.ruby-lang.org/pub/ruby/${RUBY_MAJOR%-rc}/ruby-$RUBY_VERSION.tar.gz" # Download Ruby -curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.xz -echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.xz" | sha256sum -c - +curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.gz +echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.gz" | sha256sum -c - # Skip installing Gem docs mkdir -p /usr/local/etc @@ -25,12 +25,12 @@ apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev ruby # Unpack Ruby mkdir -p /usr/src/ruby -tar -xJf ruby.tar.xz -C /usr/src/ruby --strip-components=1 +tar -xzf ruby.tar.gz -C /usr/src/ruby --strip-components=1 rm ruby.tar.xz cd /usr/src/ruby # Apply patches -find /patches/ruby/$RUBY_VERSION -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -i +find "/patches/ruby/$RUBY_VERSION" -name '*.patch' | xargs -rpn1 patch -p1 -i ./configure --enable-shared --disable-install-doc --disable-install-rdoc --disable-install-capi make install -j $(nproc) From 255e47d8874861e2d80cebbd28d208c45cd61d9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 1 Feb 2021 14:20:08 +0100 Subject: [PATCH 4/5] Fix `node-10` support --- scripts/custom-docker-build | 24 +++++++++++++----------- scripts/install-ruby | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index ba3dbf8..a3b60ff 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -148,11 +148,11 @@ function print_lfs_args() { function print_node_args() { case "$1" in - 8.x) NODE_INSTALL_VERSION=8.16.0 ;; - 10.x) NODE_INSTALL_VERSION=10.16.0 ;; - 12.x) NODE_INSTALL_VERSION=12.4.0 ;; - 12.18) NODE_INSTALL_VERSION=12.18.4 ;; - 14.15) NODE_INSTALL_VERSION=14.15.4 ;; + 8|8.x) NODE_INSTALL_VERSION=8.16.0 ;; + 10|10.x) NODE_INSTALL_VERSION=10.16.0 ;; + 12|12.x) NODE_INSTALL_VERSION=12.4.0 ;; + 12|12.18) NODE_INSTALL_VERSION=12.18.4 ;; + 14|14.15) NODE_INSTALL_VERSION=14.15.4 ;; *) echo "Unknown node version $1"; exit 1; esac printf -- "--build-arg NODE_INSTALL_VERSION=%s " "$NODE_INSTALL_VERSION" @@ -231,6 +231,7 @@ function print_pgbouncer_args() { function print_ruby_args() { case "$1" in 2.6|2.6.*) + # 2.6 requires older version of debian CUSTOM_IMAGE_NAME=debian CUSTOM_IMAGE_VERSION=stretch RUBY_VERSION="2.6.6" @@ -238,15 +239,11 @@ function print_ruby_args() { ;; 2.7|2.7.*) - CUSTOM_IMAGE_NAME=debian - CUSTOM_IMAGE_VERSION=buster RUBY_VERSION="2.7.2" RUBY_DOWNLOAD_SHA256="6e5706d0d4ee4e1e2f883db9d768586b4d06567debea353c796ec45e8321c3d4" ;; 3.0|3.0.*) - CUSTOM_IMAGE_NAME=debian - CUSTOM_IMAGE_VERSION=buster RUBY_VERSION="3.0.0" RUBY_DOWNLOAD_SHA256="a13ed141a1c18eb967aac1e33f4d6ad5f21be1ac543c344e0d6feeee54af8e28" ;; @@ -254,8 +251,6 @@ function print_ruby_args() { *) echo "Unknown ruby version $1"; exit 1; esac - printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" - printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION" printf -- "--build-arg RUBY_VERSION=%s " "$RUBY_VERSION" printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256" } @@ -263,6 +258,10 @@ function print_ruby_args() { function parse_arguments() { printf -- "-f Dockerfile.custom " + # defaults + CUSTOM_IMAGE_NAME=debian + CUSTOM_IMAGE_VERSION=buster + while read image; do read version case "$image" in @@ -282,6 +281,9 @@ function parse_arguments() { *) exit 1;; esac done + + printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" + printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION" } function generate_command() { diff --git a/scripts/install-ruby b/scripts/install-ruby index 90fd24d..6d5280f 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -26,7 +26,7 @@ apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev ruby # Unpack Ruby mkdir -p /usr/src/ruby tar -xzf ruby.tar.gz -C /usr/src/ruby --strip-components=1 -rm ruby.tar.xz +rm ruby.tar.gz cd /usr/src/ruby # Apply patches From d9530f312409f8c27e47202857db16db91e3d7b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 1 Feb 2021 18:31:37 +0100 Subject: [PATCH 5/5] Fix patching ruby --- scripts/install-ruby | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/install-ruby b/scripts/install-ruby index 6d5280f..178bc25 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -30,7 +30,14 @@ rm ruby.tar.gz cd /usr/src/ruby # Apply patches -find "/patches/ruby/$RUBY_VERSION" -name '*.patch' | xargs -rpn1 patch -p1 -i +if [[ -d "/patches/ruby/$RUBY_VERSION" ]]; then + for i in "/patches/ruby/$RUBY_VERSION"/*.patch; do + echo "$i..." + patch -p1 -i "$i" + done +fi + +# Compile ./configure --enable-shared --disable-install-doc --disable-install-rdoc --disable-install-capi make install -j $(nproc)