diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index 7454fa1..14b8a66 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -20,6 +20,27 @@ gitlab: NODE: ['16.14', '18.12'] CHROME: ['106', '108'] +gitlab-ruby-3.2: + extends: + - .build_and_push + stage: gitlab + variables: + RUBYGEMS: '3.4' + LFS: '2.9' + YARN: '1.22' + GRAPHICSMAGICK: '1.3.36' + ARCH: linux/amd64,linux/arm64 + parallel: + matrix: + - OS: ['debian:bullseye'] + RUBY: ['3.2.patched'] + GIT: ['2.36'] + POSTGRESQL: ['11', '12', '13'] + GOLANG: ['1.19'] + RUST: ['1.65.0'] + NODE: ['16.14'] + CHROME: ['108'] + # Used by GitLab's compile-production-assets and compile-test-assets jobs gitlab-assets: extends: diff --git a/Dockerfile.custom b/Dockerfile.custom index 9b6703d..b7a9adf 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -17,7 +17,12 @@ RUN /scripts/install-essentials ${BUILD_OS} ENV PATH $PATH:/usr/local/go/bin -ENV PATH /usr/local/cargo/bin:$PATH +ENV PATH /usr/local/bin:$PATH + +# Rust +ARG INSTALL_RUST_VERSION + +RUN if [ -n "$INSTALL_RUST_VERSION" ] ; then /scripts/install-rust "${INSTALL_RUST_VERSION}" && rustc --version; fi # Ruby ARG RUBY_VERSION @@ -52,11 +57,6 @@ ARG GOLANG_DOWNLOAD_SHA256 RUN if [ -n "$INSTALL_GOLANG_VERSION" ] ; then /scripts/install-golang "${INSTALL_GOLANG_VERSION}" "${GOLANG_DOWNLOAD_SHA256}" && go version; fi -# Rust -ARG INSTALL_RUST_VERSION - -RUN if [ -n "$INSTALL_RUST_VERSION" ] ; then /scripts/install-rust "${INSTALL_RUST_VERSION}" && rustc version; fi - # Git LFS (https://git-lfs.github.com/) ARG LFS_VERSION ARG LFS_DOWNLOAD_SHA256 diff --git a/patches/ruby/3.2/thread-memory-allocations.patch b/patches/ruby/3.2/thread-memory-allocations.patch new file mode 100644 index 0000000..737dfdb --- /dev/null +++ b/patches/ruby/3.2/thread-memory-allocations.patch @@ -0,0 +1,160 @@ +diff --git a/gc.c b/gc.c +index 0fccc17a32..49a4cc21b6 100644 +--- a/gc.c ++++ b/gc.c +@@ -2541,6 +2541,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, + // TODO: make it atomic, or ractor local + objspace->total_allocated_objects++; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects); ++ } ++#endif ++ + #if RGENGC_PROFILE + if (wb_protected) { + objspace->profile.total_generated_normal_object_count++; +@@ -12127,6 +12134,19 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz + #endif + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations(); ++ if (th) { ++ if (new_size > old_size) { ++ ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size); ++ } ++ ++ if (type == MEMOP_TYPE_MALLOC) { ++ ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs); ++ } ++ } ++#endif ++ + if (type == MEMOP_TYPE_MALLOC) { + retry: + if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +diff --git a/thread.c b/thread.c +index 56a5644552..176509e1be 100644 +--- a/thread.c ++++ b/thread.c +@@ -5289,6 +5289,55 @@ Init_Thread_Mutex(void) + rb_native_mutex_initialize(&th->interrupt_lock); + } + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++rb_thread_t * ++ruby_threadptr_for_trace_memory_allocations(void) ++{ ++ // The order of this checks is important due ++ // to how Ruby VM is initialized ++ if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) { ++ return GET_THREAD(); ++ } ++ ++ return NULL; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations(VALUE _) ++{ ++ return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse; ++} ++ ++static VALUE ++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val) ++{ ++ GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val); ++ return val; ++} ++ ++static VALUE ++rb_thread_memory_allocations(VALUE self) ++{ ++ rb_thread_t *th = rb_thread_ptr(self); ++ ++ if (!th->vm->thread_trace_memory_allocations) { ++ return Qnil; ++ } ++ ++ VALUE ret = rb_hash_new(); ++ ++ VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects")); ++ VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes")); ++ VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs")); ++ ++ rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects)); ++ rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes)); ++ rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs)); ++ ++ return ret; ++} ++#endif ++ + /* + * Document-class: ThreadError + * +@@ -5371,6 +5420,12 @@ Init_Thread(void) + rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0); + rb_define_alias(rb_cThread, "inspect", "to_s"); + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0); ++ rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1); ++ rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0); ++#endif ++ + rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError, + "stream closed in another thread"); + +diff --git a/vm_core.h b/vm_core.h +index 4f6e07d818..893b578cc8 100644 +--- a/vm_core.h ++++ b/vm_core.h +@@ -114,6 +114,13 @@ extern int ruby_assert_critical_section_entered; + # define VM_INSN_INFO_TABLE_IMPL 2 + #endif + ++/* ++ * track a per thread memory allocations ++ */ ++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS ++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1 ++#endif ++ + #if defined(NSIG_MAX) /* POSIX issue 8 */ + # undef NSIG + # define NSIG NSIG_MAX +@@ -661,6 +668,7 @@ typedef struct rb_vm_struct { + unsigned int thread_abort_on_exception: 1; + unsigned int thread_report_on_exception: 1; + unsigned int thread_ignore_deadlock: 1; ++ unsigned int thread_trace_memory_allocations: 1; + + /* object management */ + VALUE mark_object_ary; +@@ -1040,6 +1048,14 @@ typedef struct rb_thread_struct { + + struct rb_waiting_list *join_list; + ++#if THREAD_TRACE_MEMORY_ALLOCATIONS ++ struct { ++ size_t total_allocated_objects; ++ size_t total_malloc_bytes; ++ size_t total_mallocs; ++ } memory_allocations; ++#endif ++ + union { + struct { + VALUE proc; +@@ -1967,6 +1983,7 @@ void rb_threadptr_interrupt(rb_thread_t *th); + void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th); + void rb_threadptr_pending_interrupt_clear(rb_thread_t *th); + void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v); ++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void); + VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec); + void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo); + void rb_execution_context_update(rb_execution_context_t *ec); diff --git a/scripts/install-rust b/scripts/install-rust index d1d9f16..1356179 100755 --- a/scripts/install-rust +++ b/scripts/install-rust @@ -20,9 +20,9 @@ esac RUST_DOWNLOAD_URL="https://static.rust-lang.org/rustup/dist/$RUST_TARGET/rustup-init" RUSTUP_DEFAULT_TOOLCHAIN="$INSTALL_RUST_VERSION" -RUSTUP_HOME="/usr/local/rustup" -CARGO_HOME="/usr/local/cargo" +export RUSTUP_HOME="/opt/rust" +export CARGO_HOME="/opt/rust" function build() { curl --retry 3 --proto '=https' --tlsv1.2 -sSf "$RUST_DOWNLOAD_URL" > rustup-init @@ -37,6 +37,19 @@ function build() { rm rustup-init && rm rustup-init.sha256 chmod -R a+w "$RUSTUP_HOME" "$CARGO_HOME" + + # https://github.com/rust-lang/rustup/issues/1085 + cat < /usr/local/bin/rust-wrapper +#!/bin/sh + +RUSTUP_HOME=/opt/rust exec /opt/rust/bin/\${0##*/} "\$@" +EOF + chmod +x /usr/local/bin/rust-wrapper + + for bin in /opt/rust/bin/* + do + ln -sf /usr/local/bin/rust-wrapper /usr/local/bin/$(basename $bin) + done } build "$@" diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index edabcbd..1492f9f 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -64,13 +64,13 @@ function print_golang_args() { function print_rust_args() { case "$1" in 1.65.0) - RUST_VERSION="1.65.0" + INSTALL_RUST_VERSION="1.65.0" ;; *) fail "Unknown rust version $1" ;; esac - printf -- "--build-arg RUST_VERSION=%s " "$RUST_VERSION" + printf -- "--build-arg INSTALL_RUST_VERSION=%s " "$INSTALL_RUST_VERSION" } # If you add a new minor version here, be sure to check that the @@ -249,6 +249,12 @@ function print_ruby_args() { RUBY_DOWNLOAD_SHA256="9afc6380a027a4fe1ae1a3e2eccb6b497b9c5ac0631c12ca56f9b7beb4848776" ;; + 3.2|3.2.patched) + RUBY_VERSION="3.2.0" + RUBY_DOWNLOAD_SHA256="daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272" + ;; + + *) fail "Unknown ruby version $1" ;; esac @@ -278,6 +284,9 @@ function print_rubygems_args() { 3.2) RUBYGEMS_VERSION=3.2.33 ;; + 3.4) + RUBYGEMS_VERSION=3.4.4 + ;; *) fail "Unknown rubygems version $1" ;; esac