Merge branch 'support-using-different-base-image' into 'master'

Support using OS other than Debian as base for custom images

See merge request gitlab-org/gitlab-build-images!549

.gitlab/ci/docker.images.yml

@@ -9,11 +9,11 @@ docker:
     - .docker
     - .build_and_push
   variables:
-    DEBIAN: bullseye
+    OS: "debian:bullseye"
 
 docker-slim:
   extends:
     - .docker
     - .build_and_push
   variables:
-    DEBIAN: bullseye-slim
+    OS: "debian:bullseye-slim"

.gitlab/ci/e2e.images.yml

@@ -5,7 +5,7 @@ e2e:
   extends:
     - .build_and_push
   variables:
-    DEBIAN: bullseye
+    OS: "debian:bullseye"
     BUNDLER: '2.3'
   parallel:
     matrix:

.gitlab/ci/gitaly.images.yml

@@ -6,7 +6,7 @@ gitaly:
   stage: gitaly
   parallel:
     matrix:
-      - DEBIAN: ['bullseye']
+      - OS: ['debian:bullseye']
         RUBY: ['2.7', '3.0']
         GOLANG: ['1.16', '1.17']
         GIT: ['2.33']

.gitlab/ci/gitlab.images.yml

@@ -10,7 +10,7 @@ gitlab:
     GRAPHICSMAGICK: '1.3.36'
   parallel:
     matrix:
-      - DEBIAN: ['bullseye']
+      - OS: ['debian:bullseye']
         RUBY: ['2.7.patched', '3.0.patched']
         GIT: ['2.36']
         POSTGRESQL: ['11', '12', '13']
@@ -30,7 +30,7 @@ gitlab-assets:
     GRAPHICSMAGICK: '1.3.36'
   parallel:
     matrix:
-      - DEBIAN: ['bullseye']
+      - OS: ['debian:bullseye']
         RUBY: ['2.7', '3.0']
         GIT: ['2.33']
         NODE: ['16.14']

Dockerfile.custom

@@ -3,13 +3,17 @@
 # /scripts/custom-docker-build
 #
 
-ARG CUSTOM_IMAGE_NAME
-ARG CUSTOM_IMAGE_VERSION
-FROM ${CUSTOM_IMAGE_NAME}:${CUSTOM_IMAGE_VERSION}
+ARG CUSTOM_BASE_IMAGE
+FROM ${CUSTOM_BASE_IMAGE}
+
+# We are setting this ARG again because it is required in install-essentials
+# script. ARG defined before FROM can't be used afterwards.
+# Check https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
+ARG CUSTOM_BASE_IMAGE
 
 ADD / /
 
-RUN /scripts/install-essentials
+RUN /scripts/install-essentials ${CUSTOM_BASE_IMAGE}
 
 ENV PATH $PATH:/usr/local/go/bin
 
@@ -128,5 +132,4 @@ ENV RUBY_VERSION=${RUBY_VERSION} \
   BAZELISK_VERSION=${BAZELISK_VERSION} \
   GCLOUD_VERSION=${GCLOUD_VERSION} \
   KUBECTL_VERSION=${KUBECTL_VERSION} \
-  CUSTOM_IMAGE_NAME=${CUSTOM_IMAGE_NAME} \
-  CUSTOM_IMAGE_VERSION=${CUSTOM_IMAGE_VERSION}
+  CUSTOM_BASE_IMAGE=${CUSTOM_BASE_IMAGE}

scripts/install-chrome

@@ -3,6 +3,7 @@
 set -xeuo pipefail
 IFS=$'\n\t'
 
+function build_debian() {
     if [[ $(dpkg --print-architecture) == arm64 ]]; then
         echo "The arm64 does not have prebuilt chrome. Using chromium instead."
         apt-get update -q -y
@@ -55,3 +56,12 @@ apt-get autoremove -yq
 apt-get clean -yqq
 rm -rf /var/lib/apt/lists/*
 rm -rf /etc/apt/sources.list.d/google*.list
+}
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi

scripts/install-essentials

@@ -3,10 +3,6 @@
 set -xeuo pipefail
 IFS=$'\n\t'
 
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get update
-
 # We install `git-core` as some tooling expect `/usr/bin/git`
 # other tools that rely on PATH ordering will pick a one in `/usr/local`
 # if present
@@ -47,6 +43,11 @@ function install_debian_bullseye_deps() {
             libre2-dev libevent-dev gettext rsync git-core lsb-release
 }
 
+function prepare_debian_environment() {
+    export DEBIAN_FRONTEND=noninteractive
+
+    apt-get update
+
     VERSION=`cat /etc/issue | cut -d ' ' -f 3`
 
     case "$VERSION" in
@@ -74,3 +75,16 @@ locale -a
     apt-get autoremove -yq
     apt-get clean -yqq
     rm -rf /var/lib/apt/lists/*
+}
+
+function prepare_ubi_environment() {
+    echo "UBI preparation scripts"
+}
+
+if [[ $1 =~ debian ]]; then
+    export BUILD_OS=debian
+    prepare_debian_environment "$@"
+elif [[ $1 =~ ubi ]]; then
+    export BUILD_OS=ubi
+    prepare_ubi_environment "$@"
+fi

scripts/install-gcloud

@@ -2,10 +2,10 @@
 
 set -xeuo pipefail
 
-export DEBIAN_FRONTEND=noninteractive
-
 GCLOUD_VERSION=${1}
 
+function build_debian() {
+    export DEBIAN_FRONTEND=noninteractive
     apt-get update
     apt-get -y install \
         apt-transport-https \
@@ -23,3 +23,12 @@ apt-get install -y google-cloud-cli=${PACKAGE_VERSION}
     apt-get -yq autoremove
     apt-get clean -yqq
     rm -rf /var/lib/apt/lists/*
+}
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi

scripts/install-noto-emoji

@@ -1,6 +1,7 @@
 #!/bin/bash
 # This script installs noto color emoji
 
+function build_debian() {
     apt-get update
     apt-get install unzip
 
@@ -40,3 +41,12 @@ fc-cache -fv
     cd ..
     rm -r setup_fonts
     apt-get clean -yqq && rm -rf /var/lib/apt/lists/*
+}
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi

scripts/install-postgresql

@@ -3,6 +3,8 @@ set -xeuo pipefail
 IFS=$'\n\t'
 
 POSTGRES_VERSION=${1:-12}
+
+function build_debian() {
     DEBIAN_VERSION=$(lsb_release -c -s)
 
     export DEBIAN_FRONTEND=noninteractive
@@ -19,3 +21,12 @@ apt-get install -y postgresql-client-${POSTGRES_VERSION}
     apt-get autoremove -yq
     apt-get clean -yqq
     rm -rf /var/lib/apt/lists/*
+}
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi

scripts/install-ruby

@@ -16,6 +16,8 @@ JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${
 BUNDLER_VERSION=${3:-""}
 RUBYGEMS_VERSION=${4:-""}
 
+
+function build_debian() {
     # Install needed packages
     apt-get update
     apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf
@@ -78,6 +80,15 @@ apt-get purge -y --auto-remove ruby
     # verify we have no "ruby" packages installed
     ! dpkg -l | grep -i ruby
     [ "$(command -v ruby)" = '/usr/local/bin/ruby' ]
+}
+
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi
 
 # rough smoke test
 ruby --version

scripts/install-www-gitlab-com

@@ -3,6 +3,7 @@
 set -xeuo pipefail
 IFS=$'\n\t'
 
+function build_debian() {
     export DEBIAN_FRONTEND=noninteractive
 
     # echo "deb http://deb.debian.org/debian testing main" | tee -a /etc/apt/sources.list.d/testing.list
@@ -55,3 +56,13 @@ locale -a
     apt-get autoremove -yq
     apt-get clean -yqq
     rm -rf /var/lib/apt/lists/*
+}
+BUILD_OS=${BUILD_OS:-debian}
+
+if [[ $BUILD_OS =~ debian ]]; then
+    build_debian "$@"
+elif [[ $BUILD_OS =~ ubi ]]; then
+    build_ubi "$@"
+fi
+
+

scripts/lib/custom-docker-build

@@ -5,6 +5,14 @@ IFS=$'\n\t'
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 source "$SCRIPT_DIR/custom-docker.sh"
 
+function get_base_image_reference() {
+    if [[ $1 =~ ^debian ]]; then
+        echo "$CUSTOM_DOCKER_ARCH/$1"
+    elif [[ $1 =~ ^ubi:8 ]]; then
+        echo "registry.access.redhat.com/ubi8/$1"
+    fi
+}
+
 function print_golang_args() {
     declare -A GOLANG_DOWNLOAD_SHA256
 
@@ -321,15 +329,12 @@ function parse_arguments() {
         *) echo "unknown architecture $(arch)"; exit 1;;
     esac
 
-    CUSTOM_IMAGE_NAME=debian
-    CUSTOM_IMAGE_VERSION=buster
-
     for tool in "${PATH_TOOLS[@]}" "${TAG_TOOLS[@]}"; do
         if [ -n "${!tool}" ]; then
             version="${!tool}"
             case "$tool" in
                 ARCH) CUSTOM_DOCKER_ARCH=$version ;;
-                DEBIAN) CUSTOM_IMAGE_VERSION=$version ;;
+                OS) CUSTOM_BASE_IMAGE=get_base_image_reference $version ;;
                 RUBY) print_ruby_args $version ;;
                 BUNDLER) print_bundler_args $version ;;
                 RUBYGEMS) print_rubygems_args $version ;;
@@ -353,10 +358,11 @@ function parse_arguments() {
         fi
     done
 
-    CUSTOM_IMAGE_NAME=$CUSTOM_DOCKER_ARCH/$CUSTOM_IMAGE_NAME # ex. https://hub.docker.com/r/amd64/debian/
+    if [ -z "$CUSTOM_BASE_IMAGE" ]; then
+        CUSTOM_BASE_IMAGE="$CUSTOM_DOCKER_ARCH/debian:buster"
+    fi
 
-    printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME"
-    printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION"
+    printf -- "--build-arg CUSTOM_BASE_IMAGE=%s " "$CUSTOM_BASE_IMAGE"
 }
 
 function generate_command() {

scripts/lib/custom-docker.sh

@@ -1,13 +1,36 @@
-PATH_TOOLS=(DEBIAN RUBY GOLANG NODE POSTGRESQL)
+# Note: Check out https://wiki.bash-hackers.org/syntax/pe for documentation on
+# various variable operations used in this script.
+
+PATH_TOOLS=(OS RUBY GOLANG NODE POSTGRESQL)
 TAG_TOOLS=(BUNDLER RUBYGEMS GIT LFS CHROME YARN GRAPHICSMAGICK PGBOUNCER BAZELISK DOCKER BUILDX GCLOUD KUBECTL HELM)
 
+# Generate the docker image path using the components that were specified via
+# variables.
+# For example, consider a CI job which specifies the following variables:
+# OS: debian:bullseye
+# RUBY: 2.7
+# GOLANG: 1.16
+# GIT: 2.33
+# PGBOUNCER: 1.14
+# POSTGRESQL: 11
+# With the above variables, this function will return
+# `debian-bullseye-ruby-2.7-golang-1.16-postgresql-11`
 function get_image_path() {
     local path
     path=""
     for tool in "${PATH_TOOLS[@]}"; do
         if [[ -n "${!tool}" ]]; then
+            if [[ "${tool}" == "OS" ]]; then
+                # The OS variable's value is following <distro>:<version>
+                # format. We split that string into individual components.
+                distro=${!tool%:*}
+                version=${!tool#*:}
+                path="${path}-${distro}-${version}"
+            else
+                # Convert the tool name into lowercase using `,,` operator
                 path="${path}-${tool,,}-${!tool}"
             fi
+        fi
     done
 
     if [[ -n "$path" ]]; then
@@ -17,11 +40,22 @@ function get_image_path() {
     fi
 }
 
+# Generate the image tag using the components that were specified via variables.
+# For example, consider a CI job which specifies the following variables:
+# OS: debian:bullseye
+# RUBY: 2.7
+# GOLANG: 1.16
+# GIT: 2.33
+# PGBOUNCER: 1.14
+# POSTGRESQL: 11
+# For that job, this function will return
+# `git-2.33-pgbouncer-1.14`
 function get_image_tag() {
     local tag
     tag=""
     for tool in "${TAG_TOOLS[@]}"; do
         if [[ -n "${!tool}" ]]; then
+            # Convert the tool name into lowercase using `,,` operator
             tag="${tag}-${tool,,}-${!tool}"
         fi
     done