From a0317aa98dfabed74443033871af36debb5e0f8f Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 18:07:39 +0300 Subject: [PATCH 01/12] Add gcloud installation script --- scripts/install-gcloud | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 scripts/install-gcloud diff --git a/scripts/install-gcloud b/scripts/install-gcloud new file mode 100755 index 0000000..c6292b9 --- /dev/null +++ b/scripts/install-gcloud @@ -0,0 +1,27 @@ +#!/bin/bash + +set -xeuo pipefail + +export DEBIAN_FRONTEND=noninteractive + +GCLOUD_VERSION=${1} + +apt-get update +apt-get -y install \ + apt-transport-https \ + ca-certificates \ + curl \ + python3 \ + gnupg + +curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +apt-get update + +PACKAGE_VERSION=$(apt-cache policy google-cloud-cli | awk -v dv=${GCLOUD_VERSION} '$1 ~ dv {print $1}') + +apt-get install -y google-cloud-cli=${PACKAGE_VERSION} +apt-get -yq autoremove +apt-get clean -yqq +rm -rf /var/lib/apt/lists/* From 5cd81e9b3c38347c07b2b496d64cc7babe0b5517 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 18:23:42 +0300 Subject: [PATCH 02/12] Add kubectl install script --- scripts/install-kubectl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 scripts/install-kubectl diff --git a/scripts/install-kubectl b/scripts/install-kubectl new file mode 100755 index 0000000..18840a0 --- /dev/null +++ b/scripts/install-kubectl @@ -0,0 +1,21 @@ +#!/bin/bash + +set -xeou pipefail + +KUBECTL_VERSION=${1} +KUBECTL_DOWNLOAD_URL="https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64" + +apt-get update +apt-get -y install curl + +curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl" -o kubectl +curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl.sha256" -o kubectl.sha256 +echo "$(cat kubectl.sha256) kubectl" | sha256sum -c - + +chmod +x kubectl +mv kubectl /usr/local/bin/kubectl + +rm kubectl.sha256 +apt-get -yq autoremove +apt-get clean -yqq +rm -rf /var/lib/apt/lists/* From 7a97735026bf981239e2a6de4073e2a60b98af25 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 18:32:48 +0300 Subject: [PATCH 03/12] Integrate gcloud and kubectl in custom build --- scripts/custom-docker-build | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 00d6f30..a5efa5b 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -215,6 +215,28 @@ function print_ruby_args() { printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256" } +function print_gcloud_args() { + case "$1" in + 383) + BAZELISK_VERSION=383.0.1 + ;; + *) echo "Unknown gcloud version $1"; exit 1; + esac + + printf -- "--build-arg GCLOUD_VERSION=%s " "$GCLOUD_VERSION" +} + +function print_kubectl_args() { + case "$1" in + 1.23) + BAZELISK_VERSION=1.23.0 + ;; + *) echo "Unknown gcloud version $1"; exit 1; + esac + + printf -- "--build-arg KUBECTL_VERSION=%s " "$KUBECTL_VERSION" +} + function parse_arguments() { printf -- "-f Dockerfile.custom " @@ -239,6 +261,8 @@ function parse_arguments() { GRAPHICSMAGICK) print_graphicsmagick_args $version ;; PGBOUNCER) print_pgbouncer_args $version ;; BAZELISK) print_bazelisk_args $version ;; + GCLOUD) print_gcloud_args $version ;; + KUBECTL) print_kubectl_args $version ;; *) echo "unknown tool $tool"; exit 1;; esac fi From 5c36d00eae3e81d69f586d70f4cc83a7869a2b84 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 18:46:36 +0300 Subject: [PATCH 04/12] Add ci jobs --- .gitlab/ci/e2e.images.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .gitlab/ci/e2e.images.yml diff --git a/.gitlab/ci/e2e.images.yml b/.gitlab/ci/e2e.images.yml new file mode 100644 index 0000000..d27a7a7 --- /dev/null +++ b/.gitlab/ci/e2e.images.yml @@ -0,0 +1,21 @@ +# Used by gitlab e2e tests +.e2e: + variables: + LFS: '2.9' + GIT: '2.33' + RUBY: '2.7' + CHROME: '99' + GCLOUD: '383' + KUBECTL: '1.23' + DOCKER: '20.10.14' + +e2e test: + extends: + - .e2e + - .test_custom + +e2e push: + extends: + - .e2e + - .build_and_deploy_custom + needs: ["e2e test"] From 9906770111e31e62c27332e991d9e7070a61c6be Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 18:56:57 +0300 Subject: [PATCH 05/12] Fix copy paste mistakes in variables --- scripts/custom-docker-build | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index a5efa5b..7f3fbe7 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -218,7 +218,7 @@ function print_ruby_args() { function print_gcloud_args() { case "$1" in 383) - BAZELISK_VERSION=383.0.1 + GCLOUD_VERSION=383.0.1 ;; *) echo "Unknown gcloud version $1"; exit 1; esac @@ -229,9 +229,9 @@ function print_gcloud_args() { function print_kubectl_args() { case "$1" in 1.23) - BAZELISK_VERSION=1.23.0 + KUBECTL_VERSION=1.23.0 ;; - *) echo "Unknown gcloud version $1"; exit 1; + *) echo "Unknown kubectl version $1"; exit 1; esac printf -- "--build-arg KUBECTL_VERSION=%s " "$KUBECTL_VERSION" From 06a761ebf684520b18b5e7d60fc85c1b21a32a19 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 19:38:02 +0300 Subject: [PATCH 06/12] Update Dockerfile.custom --- Dockerfile.custom | 12 ++++++++++++ scripts/install-gcloud | 2 -- scripts/lib/custom-docker.sh | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 5e0195e..974e587 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -83,6 +83,16 @@ ARG BAZELISK_DOWNLOAD_SHA256 RUN if [ -n "$BAZELISK_VERSION" ] ; then /scripts/install-bazelisk $BAZELISK_VERSION $BAZELISK_DOWNLOAD_SHA256; fi +# GCLOUD +ARG GCLOUD_VERSION + +RUN if [ -n "$GCLOUD_VERSION" ] ; then /scripts/install-gcloud $GCLOUD_VERSION; fi + +# GCLOUD +ARG KUBECTL_VERSION + +RUN if [ -n "$KUBECTL_VERSION" ] ; then /scripts/install-kubectl $KUBECTL_VERSION; fi + RUN locale-gen C.UTF-8 ENV LANG=C.UTF-8 \ LANGUAGE=C \ @@ -101,4 +111,6 @@ ENV RUBY_VERSION=${RUBY_VERSION} \ DOCKER_VERSION=${DOCKER_VERSION} \ PGBOUNCER_VERSION=${PGBOUNCER_VERSION} \ BAZELISK_VERSION=${BAZELISK_VERSION} \ + GCLOUD_VERSION=${GCLOUD_VERSION} \ + KUBECTL_VERSION=${KUBECTL_VERSION} \ DEBIAN_VERSION=${DEBIAN_VERSION} diff --git a/scripts/install-gcloud b/scripts/install-gcloud index c6292b9..5f55b68 100755 --- a/scripts/install-gcloud +++ b/scripts/install-gcloud @@ -9,8 +9,6 @@ GCLOUD_VERSION=${1} apt-get update apt-get -y install \ apt-transport-https \ - ca-certificates \ - curl \ python3 \ gnupg diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh index e5aab5a..4d5a46d 100644 --- a/scripts/lib/custom-docker.sh +++ b/scripts/lib/custom-docker.sh @@ -1,4 +1,4 @@ -TOOLS=(DEBIAN RUBY GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK) +TOOLS=(DEBIAN RUBY GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK GCLOUD KUBECTL) function get_image_name(){ local IMAGE_NAME From 0b459c20fec88ab0d41abbced8999ba9b60d61de Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Thu, 28 Apr 2022 21:34:54 +0300 Subject: [PATCH 07/12] Remove unnecessary curl installation in kubectl --- scripts/install-kubectl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scripts/install-kubectl b/scripts/install-kubectl index 18840a0..21fa2dc 100755 --- a/scripts/install-kubectl +++ b/scripts/install-kubectl @@ -5,9 +5,6 @@ set -xeou pipefail KUBECTL_VERSION=${1} KUBECTL_DOWNLOAD_URL="https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64" -apt-get update -apt-get -y install curl - curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl" -o kubectl curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl.sha256" -o kubectl.sha256 echo "$(cat kubectl.sha256) kubectl" | sha256sum -c - @@ -16,6 +13,3 @@ chmod +x kubectl mv kubectl /usr/local/bin/kubectl rm kubectl.sha256 -apt-get -yq autoremove -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* From dd3ad9e4c97ec3c02ab9b2e7b6bcf2495a51fc35 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Fri, 29 Apr 2022 00:16:28 +0300 Subject: [PATCH 08/12] Fix copy paste mistake in comment --- Dockerfile.custom | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 974e587..ecae051 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -83,12 +83,12 @@ ARG BAZELISK_DOWNLOAD_SHA256 RUN if [ -n "$BAZELISK_VERSION" ] ; then /scripts/install-bazelisk $BAZELISK_VERSION $BAZELISK_DOWNLOAD_SHA256; fi -# GCLOUD +# Google-cloud-sdk ARG GCLOUD_VERSION RUN if [ -n "$GCLOUD_VERSION" ] ; then /scripts/install-gcloud $GCLOUD_VERSION; fi -# GCLOUD +# Kubectl ARG KUBECTL_VERSION RUN if [ -n "$KUBECTL_VERSION" ] ; then /scripts/install-kubectl $KUBECTL_VERSION; fi From 423759a39f759cb5fd1266f8f870ffd79df77132 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Fri, 29 Apr 2022 16:08:04 +0300 Subject: [PATCH 09/12] Add explicit kubectl sha256 check --- Dockerfile.custom | 3 ++- scripts/custom-docker-build | 2 ++ scripts/install-kubectl | 6 ++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index ecae051..78818ea 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -90,8 +90,9 @@ RUN if [ -n "$GCLOUD_VERSION" ] ; then /scripts/install-gcloud $GCLOUD_VERSION; # Kubectl ARG KUBECTL_VERSION +ARG KUBECTL_DOWNLOAD_SHA256 -RUN if [ -n "$KUBECTL_VERSION" ] ; then /scripts/install-kubectl $KUBECTL_VERSION; fi +RUN if [ -n "$KUBECTL_VERSION" ] ; then /scripts/install-kubectl $KUBECTL_VERSION $KUBECTL_DOWNLOAD_SHA256; fi RUN locale-gen C.UTF-8 ENV LANG=C.UTF-8 \ diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 7f3fbe7..fea1104 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -230,11 +230,13 @@ function print_kubectl_args() { case "$1" in 1.23) KUBECTL_VERSION=1.23.0 + KUBECTL_DOWNLOAD_SHA256=2d0f5ba6faa787878b642c151ccb2c3390ce4c1e6c8e2b59568b3869ba407c4f ;; *) echo "Unknown kubectl version $1"; exit 1; esac printf -- "--build-arg KUBECTL_VERSION=%s " "$KUBECTL_VERSION" + printf -- "--build-arg KUBECTL_DOWNLOAD_SHA256=%s " "$KUBECTL_DOWNLOAD_SHA256" } function parse_arguments() { diff --git a/scripts/install-kubectl b/scripts/install-kubectl index 21fa2dc..db95a52 100755 --- a/scripts/install-kubectl +++ b/scripts/install-kubectl @@ -3,13 +3,11 @@ set -xeou pipefail KUBECTL_VERSION=${1} +KUBECTL_DOWNLOAD_SHA256=${2} KUBECTL_DOWNLOAD_URL="https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64" curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl" -o kubectl -curl -fsSL "${KUBECTL_DOWNLOAD_URL}/kubectl.sha256" -o kubectl.sha256 -echo "$(cat kubectl.sha256) kubectl" | sha256sum -c - +echo "${KUBECTL_DOWNLOAD_SHA256} kubectl" | sha256sum -c - chmod +x kubectl mv kubectl /usr/local/bin/kubectl - -rm kubectl.sha256 From 34bf5cf7ee88e9890317a39fa43873f9ed800545 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Sun, 1 May 2022 14:27:13 +0300 Subject: [PATCH 10/12] Rename e2e to qa --- .gitlab/ci/{e2e.images.yml => qa.images.yml} | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename .gitlab/ci/{e2e.images.yml => qa.images.yml} (64%) diff --git a/.gitlab/ci/e2e.images.yml b/.gitlab/ci/qa.images.yml similarity index 64% rename from .gitlab/ci/e2e.images.yml rename to .gitlab/ci/qa.images.yml index d27a7a7..bfc9fe1 100644 --- a/.gitlab/ci/e2e.images.yml +++ b/.gitlab/ci/qa.images.yml @@ -1,5 +1,5 @@ -# Used by gitlab e2e tests -.e2e: +# Used by gitlab e2e tests and other qa related tasks +.qa: variables: LFS: '2.9' GIT: '2.33' @@ -9,13 +9,13 @@ KUBECTL: '1.23' DOCKER: '20.10.14' -e2e test: +qa test: extends: - - .e2e + - .qa - .test_custom -e2e push: +qa push: extends: - - .e2e + - .qa - .build_and_deploy_custom - needs: ["e2e test"] + needs: ["qa test"] From 1f15ec8684e69e888e494a67bea6e2d8f9dd55cc Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Sun, 1 May 2022 19:31:13 +0300 Subject: [PATCH 11/12] Add option to pass custom bundler version --- .gitlab/ci/qa.images.yml | 1 + Dockerfile.custom | 3 ++- scripts/custom-docker-build | 15 +++++++++++++++ scripts/install-ruby | 5 +++++ scripts/lib/custom-docker.sh | 2 +- 5 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.gitlab/ci/qa.images.yml b/.gitlab/ci/qa.images.yml index bfc9fe1..845699d 100644 --- a/.gitlab/ci/qa.images.yml +++ b/.gitlab/ci/qa.images.yml @@ -4,6 +4,7 @@ LFS: '2.9' GIT: '2.33' RUBY: '2.7' + BUNDLER: '2.3' CHROME: '99' GCLOUD: '383' KUBECTL: '1.23' diff --git a/Dockerfile.custom b/Dockerfile.custom index 78818ea..6c9e01b 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -16,8 +16,9 @@ ENV PATH $PATH:/usr/local/go/bin # Ruby ARG RUBY_VERSION ARG RUBY_DOWNLOAD_SHA256 +ARG BUNDLER_VERSION -RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 && ruby --version; fi +RUN if [ -n "$RUBY_VERSION" ]; then /scripts/install-ruby $RUBY_VERSION $RUBY_DOWNLOAD_SHA256 $BUNDLER_VERSION && ruby --version; fi # Git ARG GIT_VERSION diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index fea1104..53469ab 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -215,6 +215,20 @@ function print_ruby_args() { printf -- "--build-arg RUBY_DOWNLOAD_SHA256=%s " "$RUBY_DOWNLOAD_SHA256" } +function print_bundler_args() { + case "$1" in + 2.1) + BUNDLER_VERSION=2.1.4 + ;; + 2.3) + BUNDLER_VERSION=2.3.12 + ;; + *) echo "Unknown bundler version $1"; exit 1; + esac + + printf -- "--build-arg BUNDLER_VERSION=%s " "$BUNDLER_VERSION" +} + function print_gcloud_args() { case "$1" in 383) @@ -252,6 +266,7 @@ function parse_arguments() { case "$tool" in DEBIAN) CUSTOM_IMAGE_VERSION=$version ;; RUBY) print_ruby_args $version ;; + BUNDLER) print_bundler_args $version ;; GOLANG) print_golang_args $version ;; CHROME) print_chrome_args $version ;; DOCKER) print_docker_args $version ;; diff --git a/scripts/install-ruby b/scripts/install-ruby index ced00d9..38dfebb 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -13,6 +13,8 @@ JEMALLOC_VERSION=5.2.1 JEMALLOC_DOWNLOAD_SHA256="34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6" JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2" +BUNDLER_VERSION=${3:-""} + # Install needed packages apt-get update apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf @@ -60,6 +62,9 @@ export LDFLAGS="-Wl,--no-as-needed" cflags="-fno-omit-frame-pointer" ./configure --enable-shared --with-jemalloc --disable-install-doc --disable-install-rdoc --disable-install-capi make install -j $(nproc) +# Install specific version of bundler if provided +if [[ "$BUNDLER_VERSION" != "" ]]; then gem install bundler -v $BUNDLER_VERSION; fi + # Cleanup cd / rm -rf /usr/src/ruby /usr/src/jemalloc diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh index 4d5a46d..a790ca9 100644 --- a/scripts/lib/custom-docker.sh +++ b/scripts/lib/custom-docker.sh @@ -1,4 +1,4 @@ -TOOLS=(DEBIAN RUBY GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK GCLOUD KUBECTL) +TOOLS=(DEBIAN RUBY BUNDLER GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK GCLOUD KUBECTL) function get_image_name(){ local IMAGE_NAME From 9822179497af810c17d7f07ed0d85a5c7c5f73f8 Mon Sep 17 00:00:00 2001 From: Andrejs Cunskis Date: Sun, 1 May 2022 22:29:14 +0300 Subject: [PATCH 12/12] Set debian and add missing docker for custom build --- .gitlab/ci/qa.images.yml | 1 + scripts/lib/custom-docker.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/qa.images.yml b/.gitlab/ci/qa.images.yml index 845699d..4e1d836 100644 --- a/.gitlab/ci/qa.images.yml +++ b/.gitlab/ci/qa.images.yml @@ -1,6 +1,7 @@ # Used by gitlab e2e tests and other qa related tasks .qa: variables: + DEBIAN: bullseye LFS: '2.9' GIT: '2.33' RUBY: '2.7' diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh index a790ca9..91a8978 100644 --- a/scripts/lib/custom-docker.sh +++ b/scripts/lib/custom-docker.sh @@ -1,4 +1,4 @@ -TOOLS=(DEBIAN RUBY BUNDLER GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK GCLOUD KUBECTL) +TOOLS=(DEBIAN RUBY BUNDLER GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK DOCKER GCLOUD KUBECTL) function get_image_name(){ local IMAGE_NAME