Merge branch 'acunskis-gcloud-kubectl' into 'master'

E2E: Custom docker image for gitlab e2e tests

See merge request gitlab-org/gitlab-build-images!525

.gitlab/ci/qa.images.yml

@@ -0,0 +1,23 @@
+# Used by gitlab e2e tests and other qa related tasks
+.qa:
+  variables:
+    DEBIAN: bullseye
+    LFS: '2.9'
+    GIT: '2.33'
+    RUBY: '2.7'
+    BUNDLER: '2.3'
+    CHROME: '99'
+    GCLOUD: '383'
+    KUBECTL: '1.23'
+    DOCKER: '20.10.14'
+
+qa test:
+  extends:
+    - .qa
+    - .test_custom
+
+qa push:
+  extends:
+    - .qa
+    - .build_and_deploy_custom
+  needs: ["qa test"]

Dockerfile.custom

@@ -16,8 +16,9 @@ ENV PATH $PATH:/usr/local/go/bin
 # Ruby
 ARG RUBY_VERSION
 ARG RUBY_DOWNLOAD_SHA256
+ARG BUNDLER_VERSION
 
-RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 && ruby --version; fi
+RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 $BUNDLER_VERSION && ruby --version; fi
 
 # Git
 ARG GIT_VERSION
@@ -83,6 +84,17 @@ ARG BAZELISK_DOWNLOAD_SHA256
 
 RUN if [ -n "$BAZELISK_VERSION" ] ; then /scripts/install-bazelisk $BAZELISK_VERSION $BAZELISK_DOWNLOAD_SHA256; fi
 
+# Google-cloud-sdk
+ARG GCLOUD_VERSION
+
+RUN if [ -n "$GCLOUD_VERSION" ] ; then /scripts/install-gcloud $GCLOUD_VERSION; fi
+
+# Kubectl
+ARG KUBECTL_VERSION
+ARG KUBECTL_DOWNLOAD_SHA256
+
+RUN if [ -n "$KUBECTL_VERSION" ] ; then /scripts/install-kubectl $KUBECTL_VERSION $KUBECTL_DOWNLOAD_SHA256; fi
+
 RUN locale-gen C.UTF-8
 ENV LANG=C.UTF-8 \
   LANGUAGE=C \
@@ -101,4 +113,6 @@ ENV RUBY_VERSION=${RUBY_VERSION} \
   DOCKER_VERSION=${DOCKER_VERSION} \
   PGBOUNCER_VERSION=${PGBOUNCER_VERSION} \
   BAZELISK_VERSION=${BAZELISK_VERSION} \
+  GCLOUD_VERSION=${GCLOUD_VERSION} \
+  KUBECTL_VERSION=${KUBECTL_VERSION} \
   DEBIAN_VERSION=${DEBIAN_VERSION}

scripts/custom-docker-build

@@ -215,6 +215,44 @@ function print_ruby_args() {
     printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256"
 }
 
+function print_bundler_args() {
+    case "$1" in
+        2.1)
+            BUNDLER_VERSION=2.1.4
+            ;;
+        2.3)
+            BUNDLER_VERSION=2.3.12
+            ;;
+        *) echo "Unknown bundler version $1"; exit 1;
+    esac
+
+    printf -- "--build-arg BUNDLER_VERSION=%s " "$BUNDLER_VERSION"
+}
+
+function print_gcloud_args() {
+    case "$1" in
+        383)
+            GCLOUD_VERSION=383.0.1
+            ;;
+        *) echo "Unknown gcloud version $1"; exit 1;
+    esac
+
+    printf -- "--build-arg GCLOUD_VERSION=%s " "$GCLOUD_VERSION"
+}
+
+function print_kubectl_args() {
+    case "$1" in
+        1.23)
+            KUBECTL_VERSION=1.23.0
+            KUBECTL_DOWNLOAD_SHA256=2d0f5ba6faa787878b642c151ccb2c3390ce4c1e6c8e2b59568b3869ba407c4f
+            ;;
+        *) echo "Unknown kubectl version $1"; exit 1;
+    esac
+
+    printf -- "--build-arg KUBECTL_VERSION=%s " "$KUBECTL_VERSION"
+    printf -- "--build-arg KUBECTL_DOWNLOAD_SHA256=%s " "$KUBECTL_DOWNLOAD_SHA256"
+}
+
 function parse_arguments() {
     printf -- "-f Dockerfile.custom "
 
@@ -228,6 +266,7 @@ function parse_arguments() {
             case "$tool" in
                 DEBIAN) CUSTOM_IMAGE_VERSION=$version ;;
                 RUBY) print_ruby_args $version ;;
+                BUNDLER) print_bundler_args $version ;;
                 GOLANG) print_golang_args $version ;;
                 CHROME) print_chrome_args $version ;;
                 DOCKER) print_docker_args $version ;;
@@ -239,6 +278,8 @@ function parse_arguments() {
                 GRAPHICSMAGICK) print_graphicsmagick_args $version ;;
                 PGBOUNCER) print_pgbouncer_args $version ;;
                 BAZELISK) print_bazelisk_args $version ;;
+                GCLOUD) print_gcloud_args $version ;;
+                KUBECTL) print_kubectl_args $version ;;
                 *) echo "unknown tool $tool"; exit 1;;
             esac
         fi

scripts/install-gcloud

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+export DEBIAN_FRONTEND=noninteractive
+
+GCLOUD_VERSION=${1}
+
+apt-get update
+apt-get -y install \
+    apt-transport-https \
+    python3 \
+    gnupg
+
+curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+
+apt-get update
+
+PACKAGE_VERSION=$(apt-cache policy google-cloud-cli | awk -v dv=${GCLOUD_VERSION} '$1 ~ dv {print $1}')
+
+apt-get install -y google-cloud-cli=${PACKAGE_VERSION}
+apt-get -yq autoremove
+apt-get clean -yqq
+rm -rf /var/lib/apt/lists/*

scripts/install-kubectl

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -xeou pipefail
+
+KUBECTL_VERSION=${1}
+KUBECTL_DOWNLOAD_SHA256=${2}
+KUBECTL_DOWNLOAD_URL="https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64"
+
+curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl" -o kubectl
+echo "${KUBECTL_DOWNLOAD_SHA256}  kubectl" | sha256sum -c -
+
+chmod +x kubectl
+mv kubectl /usr/local/bin/kubectl

scripts/install-ruby

@@ -13,6 +13,8 @@ JEMALLOC_VERSION=5.2.1
 JEMALLOC_DOWNLOAD_SHA256="34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6"
 JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
 
+BUNDLER_VERSION=${3:-""}
+
 # Install needed packages
 apt-get update
 apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf
@@ -60,6 +62,9 @@ export LDFLAGS="-Wl,--no-as-needed"
 cflags="-fno-omit-frame-pointer" ./configure --enable-shared --with-jemalloc --disable-install-doc --disable-install-rdoc --disable-install-capi
 make install -j $(nproc)
 
+# Install specific version of bundler if provided
+if [[ "$BUNDLER_VERSION" != "" ]]; then gem install bundler -v $BUNDLER_VERSION; fi
+
 # Cleanup
 cd /
 rm -rf /usr/src/ruby /usr/src/jemalloc

scripts/lib/custom-docker.sh

@@ -1,4 +1,4 @@
-TOOLS=(DEBIAN RUBY GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK)
+TOOLS=(DEBIAN RUBY BUNDLER GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK DOCKER GCLOUD KUBECTL)
 
 function get_image_name(){
     local IMAGE_NAME