Merge branch 'add-ubi-image-for-gitaly' into 'master'

Add UBI variants of Gitaly images

See merge request gitlab-org/gitlab-build-images!559

.gitlab/ci/gitaly.images.yml

@@ -10,3 +10,7 @@ gitaly:
         RUBY: ['2.7', '3.0']
         GOLANG: ['1.17', '1.18']
         GIT: ['2.36']
+      - OS: ['ubi:8.6']
+        RUBY: ['2.7']
+        GOLANG: ['1.17']
+        GIT: ['2.36']

Dockerfile.custom

@@ -6,14 +6,12 @@
 ARG CUSTOM_BASE_IMAGE
 FROM ${CUSTOM_BASE_IMAGE}
 
-# We are setting this ARG again because it is required in install-essentials
+ARG BUILD_OS
-# script. ARG defined before FROM can't be used afterwards.
+ARG BUILD_ARCH
-# Check https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
-ARG CUSTOM_BASE_IMAGE
 
 ADD / /
 
-RUN /scripts/install-essentials ${CUSTOM_BASE_IMAGE}
+RUN /scripts/install-essentials ${BUILD_OS}
 
 ENV PATH $PATH:/usr/local/go/bin
 
@@ -102,7 +100,7 @@ ARG HELM_DOWNLOAD_SHA256
 
 RUN if [ -n "$HELM_VERSION" ] ; then /scripts/install-helm $HELM_VERSION $HELM_DOWNLOAD_SHA256; fi
 
-RUN locale-gen C.UTF-8
+RUN /scripts/generate-locale
 ENV LANG=C.UTF-8 \
   LANGUAGE=C \
   LC_ALL=C.UTF-8
@@ -123,5 +121,4 @@ ENV RUBY_VERSION=${RUBY_VERSION} \
   DOCKER_VERSION=${DOCKER_VERSION} \
   BAZELISK_VERSION=${BAZELISK_VERSION} \
   GCLOUD_VERSION=${GCLOUD_VERSION} \
-  KUBECTL_VERSION=${KUBECTL_VERSION} \
+  KUBECTL_VERSION=${KUBECTL_VERSION}
-  CUSTOM_BASE_IMAGE=${CUSTOM_BASE_IMAGE}

scripts/generate-locale

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -xeou pipefail
+
+BUILD_OS=${BUILD_OS:-debian}
+
+# For UBI, the locale is already present in the image.
+if [[ $BUILD_OS =~ debian ]]; then
+    locale-gen C.UTF-8
+fi

scripts/install-essentials

@@ -78,13 +78,21 @@ function prepare_debian_environment() {
 }
 
 function prepare_ubi_environment() {
-    echo "UBI preparation scripts"
+    yum update -y
+
+    yum install -by --nodocs \
+                autoconf cmake gcc gcc-c++ make patch perl bzip2 \
+                libedit ncurses uuid libarchive curl-devel \
+                libicu-devel libffi-devel libuuid-devel openssl-devel \
+                ncurses-devel pcre2-devel zlib-devel libstdc++-static \
+                libevent-devel redhat-lsb-core procps-ng
+
+    yum autoremove -y
+    yum clean -y all
 }
 
 if [[ $1 =~ debian ]]; then
-    export BUILD_OS=debian
     prepare_debian_environment "$@"
 elif [[ $1 =~ ubi ]]; then
-    export BUILD_OS=ubi
     prepare_ubi_environment "$@"
 fi

scripts/install-golang

@@ -5,9 +5,44 @@ set -xeou pipefail
 INSTALL_GOLANG_VERSION=${1}
 GOLANG_DOWNLOAD_SHA256=${2}
 
-GOLANG_DOWNLOAD_URL="https://golang.org/dl/go${INSTALL_GOLANG_VERSION}.linux-$(dpkg --print-architecture).tar.gz"
+GOLANG_DOWNLOAD_URL="https://golang.org/dl/go${INSTALL_GOLANG_VERSION}.linux-${BUILD_ARCH}.tar.gz"
 
-curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz
+function build_debian() {
-echo "${GOLANG_DOWNLOAD_SHA256}  golang.tar.gz" | sha256sum -c -
+    curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz
-tar -C /usr/local -xzf golang.tar.gz
+    echo "${GOLANG_DOWNLOAD_SHA256}  golang.tar.gz" | sha256sum -c -
-rm golang.tar.gz
+    tar -C /usr/local -xzf golang.tar.gz
+    rm golang.tar.gz
+}
+
+function build_ubi() {
+    GO_MAJOR_VERSION=${INSTALL_GOLANG_VERSION%.*}
+    GOLANG_FIPS_BRANCH="go${GO_MAJOR_VERSION}-openssl-fips"
+
+    mkdir -p /tmp/golang
+    curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz
+    echo "${GOLANG_DOWNLOAD_SHA256}  golang.tar.gz" | sha256sum -c -
+    tar -C /tmp/golang -xzf golang.tar.gz
+
+    # For UBI, we will be installing golang-fips
+    git clone https://github.com/golang-fips/go.git --branch ${GOLANG_FIPS_BRANCH} --single-branch --depth 1 /usr/local/go
+
+    cd /usr/local/go/src
+
+    PATH=$PATH:/tmp/golang/go/bin CGO_ENABLED=1 ./make.bash
+
+    rm -rf /usr/local/go/pkg/*/cmd /usr/local/go/pkg/bootstrap \
+           /usr/local/go/pkg/obj /usr/local/go/pkg/tool/*/api \
+           /usr/local/go/pkg/tool/*/go_bootstrap /usr/local/go/src/cmd/dist/dist \
+           /usr/local/go/.git* /tmp/golang
+
+    ln -sf /usr/local/go/bin/go /usr/local/go/bin/gofmt /usr/local/go/bin/godoc /usr/local/bin/
+}
+
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi

scripts/install-ruby

@@ -16,12 +16,7 @@ JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${
 BUNDLER_VERSION=${3:-""}
 RUBYGEMS_VERSION=${4:-""}
 
-
+function build_common() {
-function build_debian() {
-    # Install needed packages
-    apt-get update
-    apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf
-
     # Download jemalloc
     mkdir -p /usr/src/jemalloc
     cd /usr/src/jemalloc
@@ -74,6 +69,15 @@ function build_debian() {
     # Cleanup
     cd /
     rm -rf /usr/src/ruby /usr/src/jemalloc
+}
+
+function build_debian() {
+    # Install needed packages
+    apt-get update
+    apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf
+
+    build_common
+
     apt-get purge -y --auto-remove ruby
 
     # Verify
@@ -82,6 +86,21 @@ function build_debian() {
     [ "$(command -v ruby)" = '/usr/local/bin/ruby' ]
 }
 
+function build_ubi() {
+    yum update -y
+
+    build_common
+
+    yum remove -y ruby
+    yum autoremove -y
+    yum clean -y all
+
+    # Verify
+    # verify we have no "ruby" packages installed
+    ! yum list installed | grep -i ruby
+    [ "$(command -v ruby)" = '/usr/local/bin/ruby' ]
+}
+
 BUILD_OS=${BUILD_OS:-debian}
 
 if [[ $BUILD_OS =~ debian ]]; then

scripts/lib/custom-docker-build

@@ -301,10 +301,32 @@ function parse_arguments() {
     # armhf   | armhf            | armhf                     | arm32v7
     # aarch64 | aarch64          | arm64                     | arm64v8
 
+    # We make use of 4 variables related to the build environment.
+    # 1. `CUSTOM_DOCKER_IMAGE` - Defines the reference of image used as
+    #                            base for building images. Follows a
+    #                            valid docker image reference format
+    #                              - "arm64v8/debian:buster"
+    #                              - "registry.access.redhat.com/ubi8/ubi:8.6"
+    # 2. `CUSTOM_DOCKER_ARCH`  - Defines the architecture of the docker
+    #                            image. Follows docker architecture
+    #                            format - `amd64`/`arm32v7`/`arm64v8`.
+    # 3. `BUILD_OS`            - Used by installation scripts of various
+    #                            components to detect the OS being worked
+    #                            on. Follows regular docker image:tag
+    #                            format - "debian:buster" or "ubi:8.6".
+    # 4. `BUILD_ARCH`          - Used by installation scripts of various
+    #                            components to detect the architecture
+    #                            being worked on. Follows general
+    #                            architecture names - `amd64`/`arm64`.
+    #
+    # The first two variables are used to pick the docker image used as
+    # base for builds. The last two variables are used to detect OS and
+    # architecture while building components.
+
     # defaults
     case $(arch) in
-        x86_64) CUSTOM_DOCKER_ARCH=amd64 ;;
+        x86_64) CUSTOM_DOCKER_ARCH=amd64; BUILD_ARCH=amd64 ;;
-        aarch64) CUSTOM_DOCKER_ARCH=arm64v8 ;;
+        aarch64) CUSTOM_DOCKER_ARCH=arm64v8; BUILD_ARCH=arm64 ;;
         *) echo "unknown architecture $(arch)"; exit 1;;
     esac
 
@@ -340,7 +362,11 @@ function parse_arguments() {
         CUSTOM_BASE_IMAGE="$CUSTOM_DOCKER_ARCH/debian:buster"
     fi
 
+    BUILD_OS=${OS:-"debian:buster"}
+
     printf -- "--build-arg CUSTOM_BASE_IMAGE=%s " "$CUSTOM_BASE_IMAGE"
+    printf -- "--build-arg BUILD_OS=%s " "$BUILD_OS"
+    printf -- "--build-arg BUILD_ARCH=%s " "$BUILD_ARCH"
 }
 
 function generate_command() {