Merge branch 'sh-add-ruby-3.2' into 'master'

Add Ruby 3.2.0 images and fix Rust installation

See merge request https://gitlab.com/gitlab-org/gitlab-build-images/-/merge_requests/625

Merged-by: Stan Hu <stanhu@gmail.com>
Approved-by: Balasankar 'Balu' C <balasankar@gitlab.com>

.gitlab/ci/gitlab.images.yml

@@ -20,6 +20,27 @@ gitlab:
         NODE: ['16.14', '18.12']
         CHROME: ['106', '108']
 
+gitlab-ruby-3.2:
+  extends:
+    - .build_and_push
+  stage: gitlab
+  variables:
+    RUBYGEMS: '3.4'
+    LFS: '2.9'
+    YARN: '1.22'
+    GRAPHICSMAGICK: '1.3.36'
+    ARCH: linux/amd64,linux/arm64
+  parallel:
+    matrix:
+      - OS: ['debian:bullseye']
+        RUBY: ['3.2.patched']
+        GIT: ['2.36']
+        POSTGRESQL: ['11', '12', '13']
+        GOLANG: ['1.19']
+        RUST: ['1.65.0']
+        NODE: ['16.14']
+        CHROME: ['108']
+
 # Used by GitLab's compile-production-assets and compile-test-assets jobs
 gitlab-assets:
   extends:

Dockerfile.custom

@@ -17,7 +17,12 @@ RUN /scripts/install-essentials ${BUILD_OS}
 
 ENV PATH $PATH:/usr/local/go/bin
 
-ENV PATH /usr/local/cargo/bin:$PATH
+ENV PATH /usr/local/bin:$PATH
+
+# Rust
+ARG INSTALL_RUST_VERSION
+
+RUN if [ -n "$INSTALL_RUST_VERSION" ] ; then /scripts/install-rust "${INSTALL_RUST_VERSION}" && rustc --version; fi
 
 # Ruby
 ARG RUBY_VERSION
@@ -52,11 +57,6 @@ ARG GOLANG_DOWNLOAD_SHA256
 
 RUN if [ -n "$INSTALL_GOLANG_VERSION" ] ; then /scripts/install-golang "${INSTALL_GOLANG_VERSION}" "${GOLANG_DOWNLOAD_SHA256}" && go version; fi
 
-# Rust
-ARG INSTALL_RUST_VERSION
-
-RUN if [ -n "$INSTALL_RUST_VERSION" ] ; then /scripts/install-rust "${INSTALL_RUST_VERSION}" && rustc version; fi
-
 # Git LFS (https://git-lfs.github.com/)
 ARG LFS_VERSION
 ARG LFS_DOWNLOAD_SHA256

patches/ruby/3.2/thread-memory-allocations.patch

@@ -0,0 +1,160 @@
+diff --git a/gc.c b/gc.c
+index 0fccc17a32..49a4cc21b6 100644
+--- a/gc.c
++++ b/gc.c
+@@ -2541,6 +2541,13 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
+     // TODO: make it atomic, or ractor local
+     objspace->total_allocated_objects++;
+ 
++#if THREAD_TRACE_MEMORY_ALLOCATIONS
++    rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
++    if (th) {
++	ATOMIC_SIZE_INC(th->memory_allocations.total_allocated_objects);
++    }
++#endif
++
+ #if RGENGC_PROFILE
+     if (wb_protected) {
+         objspace->profile.total_generated_normal_object_count++;
+@@ -12127,6 +12134,19 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz
+ #endif
+     }
+ 
++#if THREAD_TRACE_MEMORY_ALLOCATIONS
++    rb_thread_t *th = ruby_threadptr_for_trace_memory_allocations();
++    if (th) {
++	if (new_size > old_size) {
++	    ATOMIC_SIZE_ADD(th->memory_allocations.total_malloc_bytes, new_size - old_size);
++	}
++
++	if (type == MEMOP_TYPE_MALLOC) {
++	    ATOMIC_SIZE_INC(th->memory_allocations.total_mallocs);
++	}
++    }
++#endif
++
+     if (type == MEMOP_TYPE_MALLOC) {
+       retry:
+         if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) {
+diff --git a/thread.c b/thread.c
+index 56a5644552..176509e1be 100644
+--- a/thread.c
++++ b/thread.c
+@@ -5289,6 +5289,55 @@ Init_Thread_Mutex(void)
+     rb_native_mutex_initialize(&th->interrupt_lock);
+ }
+ 
++#if THREAD_TRACE_MEMORY_ALLOCATIONS
++rb_thread_t *
++ruby_threadptr_for_trace_memory_allocations(void)
++{
++    // The order of this checks is important due
++    // to how Ruby VM is initialized
++    if (GET_VM()->thread_trace_memory_allocations && GET_EC() != NULL) {
++        return GET_THREAD();
++    }
++
++    return NULL;
++}
++
++static VALUE
++rb_thread_s_trace_memory_allocations(VALUE _)
++{
++    return GET_THREAD()->vm->thread_trace_memory_allocations ? Qtrue : Qfalse;
++}
++
++static VALUE
++rb_thread_s_trace_memory_allocations_set(VALUE self, VALUE val)
++{
++    GET_THREAD()->vm->thread_trace_memory_allocations = RTEST(val);
++    return val;
++}
++
++static VALUE
++rb_thread_memory_allocations(VALUE self)
++{
++    rb_thread_t *th = rb_thread_ptr(self);
++
++    if (!th->vm->thread_trace_memory_allocations) {
++        return Qnil;
++    }
++
++    VALUE ret = rb_hash_new();
++
++	VALUE total_allocated_objects = ID2SYM(rb_intern_const("total_allocated_objects"));
++	VALUE total_malloc_bytes = ID2SYM(rb_intern_const("total_malloc_bytes"));
++	VALUE total_mallocs = ID2SYM(rb_intern_const("total_mallocs"));
++
++	rb_hash_aset(ret, total_allocated_objects, SIZET2NUM(th->memory_allocations.total_allocated_objects));
++	rb_hash_aset(ret, total_malloc_bytes, SIZET2NUM(th->memory_allocations.total_malloc_bytes));
++	rb_hash_aset(ret, total_mallocs, SIZET2NUM(th->memory_allocations.total_mallocs));
++
++    return ret;
++}
++#endif
++
+ /*
+  *  Document-class: ThreadError
+  *
+@@ -5371,6 +5420,12 @@ Init_Thread(void)
+     rb_define_method(rb_cThread, "to_s", rb_thread_to_s, 0);
+     rb_define_alias(rb_cThread, "inspect", "to_s");
+ 
++#if THREAD_TRACE_MEMORY_ALLOCATIONS
++    rb_define_singleton_method(rb_cThread, "trace_memory_allocations", rb_thread_s_trace_memory_allocations, 0);
++    rb_define_singleton_method(rb_cThread, "trace_memory_allocations=", rb_thread_s_trace_memory_allocations_set, 1);
++    rb_define_method(rb_cThread, "memory_allocations", rb_thread_memory_allocations, 0);
++#endif
++
+     rb_vm_register_special_exception(ruby_error_stream_closed, rb_eIOError,
+                                      "stream closed in another thread");
+ 
+diff --git a/vm_core.h b/vm_core.h
+index 4f6e07d818..893b578cc8 100644
+--- a/vm_core.h
++++ b/vm_core.h
+@@ -114,6 +114,13 @@ extern int ruby_assert_critical_section_entered;
+ # define VM_INSN_INFO_TABLE_IMPL 2
+ #endif
+ 
++/*
++ * track a per thread memory allocations
++ */
++#ifndef THREAD_TRACE_MEMORY_ALLOCATIONS
++# define THREAD_TRACE_MEMORY_ALLOCATIONS 1
++#endif
++
+ #if defined(NSIG_MAX)           /* POSIX issue 8 */
+ # undef NSIG
+ # define NSIG NSIG_MAX
+@@ -661,6 +668,7 @@ typedef struct rb_vm_struct {
+     unsigned int thread_abort_on_exception: 1;
+     unsigned int thread_report_on_exception: 1;
+     unsigned int thread_ignore_deadlock: 1;
++    unsigned int thread_trace_memory_allocations: 1;
+ 
+     /* object management */
+     VALUE mark_object_ary;
+@@ -1040,6 +1048,14 @@ typedef struct rb_thread_struct {
+ 
+     struct rb_waiting_list *join_list;
+ 
++#if THREAD_TRACE_MEMORY_ALLOCATIONS
++    struct {
++        size_t total_allocated_objects;
++        size_t total_malloc_bytes;
++        size_t total_mallocs;
++    } memory_allocations;
++#endif
++
+     union {
+         struct {
+             VALUE proc;
+@@ -1967,6 +1983,7 @@ void rb_threadptr_interrupt(rb_thread_t *th);
+ void rb_threadptr_unlock_all_locking_mutexes(rb_thread_t *th);
+ void rb_threadptr_pending_interrupt_clear(rb_thread_t *th);
+ void rb_threadptr_pending_interrupt_enque(rb_thread_t *th, VALUE v);
++rb_thread_t *ruby_threadptr_for_trace_memory_allocations(void);
+ VALUE rb_ec_get_errinfo(const rb_execution_context_t *ec);
+ void rb_ec_error_print(rb_execution_context_t * volatile ec, volatile VALUE errinfo);
+ void rb_execution_context_update(rb_execution_context_t *ec);

scripts/install-rust

@@ -20,9 +20,9 @@ esac
 RUST_DOWNLOAD_URL="https://static.rust-lang.org/rustup/dist/$RUST_TARGET/rustup-init"
 
 RUSTUP_DEFAULT_TOOLCHAIN="$INSTALL_RUST_VERSION"
-RUSTUP_HOME="/usr/local/rustup"
 
-CARGO_HOME="/usr/local/cargo"
+export RUSTUP_HOME="/opt/rust"
+export CARGO_HOME="/opt/rust"
 
 function build() {
     curl --retry 3 --proto '=https' --tlsv1.2 -sSf "$RUST_DOWNLOAD_URL" > rustup-init
@@ -37,6 +37,19 @@ function build() {
     rm rustup-init && rm rustup-init.sha256
 
     chmod -R a+w "$RUSTUP_HOME" "$CARGO_HOME"
+
+    # https://github.com/rust-lang/rustup/issues/1085
+    cat <<EOF > /usr/local/bin/rust-wrapper
+#!/bin/sh
+
+RUSTUP_HOME=/opt/rust exec /opt/rust/bin/\${0##*/} "\$@"
+EOF
+    chmod +x /usr/local/bin/rust-wrapper
+
+    for bin in /opt/rust/bin/*
+    do
+	ln -sf /usr/local/bin/rust-wrapper /usr/local/bin/$(basename $bin)
+    done
 }
 
 build "$@"

scripts/lib/custom-docker-build

@@ -64,13 +64,13 @@ function print_golang_args() {
 function print_rust_args() {
     case "$1" in
         1.65.0)
-            RUST_VERSION="1.65.0"
+            INSTALL_RUST_VERSION="1.65.0"
             ;;
 
         *) fail "Unknown rust version $1" ;;
     esac
 
-    printf -- "--build-arg RUST_VERSION=%s " "$RUST_VERSION"
+    printf -- "--build-arg INSTALL_RUST_VERSION=%s " "$INSTALL_RUST_VERSION"
 }
 
 # If you add a new minor version here, be sure to check that the
@@ -249,6 +249,12 @@ function print_ruby_args() {
             RUBY_DOWNLOAD_SHA256="9afc6380a027a4fe1ae1a3e2eccb6b497b9c5ac0631c12ca56f9b7beb4848776"
             ;;
 
+        3.2|3.2.patched)
+            RUBY_VERSION="3.2.0"
+            RUBY_DOWNLOAD_SHA256="daaa78e1360b2783f98deeceb677ad900f3a36c0ffa6e2b6b19090be77abc272"
+            ;;
+
+
         *) fail "Unknown ruby version $1" ;;
     esac
 
@@ -278,6 +284,9 @@ function print_rubygems_args() {
         3.2)
             RUBYGEMS_VERSION=3.2.33
             ;;
+        3.4)
+            RUBYGEMS_VERSION=3.4.4
+            ;;
         *) fail "Unknown rubygems version $1" ;;
     esac