From 0dea43ac3b01800955d96f17f66e1f9e28d6d970 Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Wed, 15 Jun 2022 09:51:03 +0530 Subject: [PATCH] Support using OS other than Debian as base for custom images Signed-off-by: Balasankar "Balu" C --- .gitlab/ci/docker.images.yml | 4 +- .gitlab/ci/e2e.images.yml | 2 +- .gitlab/ci/gitaly.images.yml | 2 +- .gitlab/ci/gitlab.images.yml | 4 +- Dockerfile.custom | 15 ++-- scripts/install-chrome | 50 +++++++------ scripts/install-essentials | 70 ++++++++++-------- scripts/install-gcloud | 39 ++++++---- scripts/install-noto-emoji | 76 +++++++++++--------- scripts/install-postgresql | 33 ++++++--- scripts/install-ruby | 121 +++++++++++++++++--------------- scripts/install-www-gitlab-com | 95 ++++++++++++++----------- scripts/lib/custom-docker-build | 20 ++++-- scripts/lib/custom-docker.sh | 40 ++++++++++- 14 files changed, 345 insertions(+), 226 deletions(-) diff --git a/.gitlab/ci/docker.images.yml b/.gitlab/ci/docker.images.yml index c8fc67a..96b72c8 100644 --- a/.gitlab/ci/docker.images.yml +++ b/.gitlab/ci/docker.images.yml @@ -9,11 +9,11 @@ docker: - .docker - .build_and_push variables: - DEBIAN: bullseye + OS: "debian:bullseye" docker-slim: extends: - .docker - .build_and_push variables: - DEBIAN: bullseye-slim + OS: "debian:bullseye-slim" diff --git a/.gitlab/ci/e2e.images.yml b/.gitlab/ci/e2e.images.yml index aca6f9d..ef8652b 100644 --- a/.gitlab/ci/e2e.images.yml +++ b/.gitlab/ci/e2e.images.yml @@ -5,7 +5,7 @@ e2e: extends: - .build_and_push variables: - DEBIAN: bullseye + OS: "debian:bullseye" BUNDLER: '2.3' parallel: matrix: diff --git a/.gitlab/ci/gitaly.images.yml b/.gitlab/ci/gitaly.images.yml index 4896b4e..ec0c05b 100644 --- a/.gitlab/ci/gitaly.images.yml +++ b/.gitlab/ci/gitaly.images.yml @@ -6,7 +6,7 @@ gitaly: stage: gitaly parallel: matrix: - - DEBIAN: ['bullseye'] + - OS: ['debian:bullseye'] RUBY: ['2.7', '3.0'] GOLANG: ['1.16', '1.17'] GIT: ['2.33'] diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index 88a40b2..4180ab0 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -10,7 +10,7 @@ gitlab: GRAPHICSMAGICK: '1.3.36' parallel: matrix: - - DEBIAN: ['bullseye'] + - OS: ['debian:bullseye'] RUBY: ['2.7.patched', '3.0.patched'] GIT: ['2.36'] POSTGRESQL: ['11', '12', '13'] @@ -30,7 +30,7 @@ gitlab-assets: GRAPHICSMAGICK: '1.3.36' parallel: matrix: - - DEBIAN: ['bullseye'] + - OS: ['debian:bullseye'] RUBY: ['2.7', '3.0'] GIT: ['2.33'] NODE: ['16.14'] diff --git a/Dockerfile.custom b/Dockerfile.custom index f962367..79e4ffe 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -3,13 +3,17 @@ # /scripts/custom-docker-build # -ARG CUSTOM_IMAGE_NAME -ARG CUSTOM_IMAGE_VERSION -FROM ${CUSTOM_IMAGE_NAME}:${CUSTOM_IMAGE_VERSION} +ARG CUSTOM_BASE_IMAGE +FROM ${CUSTOM_BASE_IMAGE} + +# We are setting this ARG again because it is required in install-essentials +# script. ARG defined before FROM can't be used afterwards. +# Check https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact +ARG CUSTOM_BASE_IMAGE ADD / / -RUN /scripts/install-essentials +RUN /scripts/install-essentials ${CUSTOM_BASE_IMAGE} ENV PATH $PATH:/usr/local/go/bin @@ -128,5 +132,4 @@ ENV RUBY_VERSION=${RUBY_VERSION} \ BAZELISK_VERSION=${BAZELISK_VERSION} \ GCLOUD_VERSION=${GCLOUD_VERSION} \ KUBECTL_VERSION=${KUBECTL_VERSION} \ - CUSTOM_IMAGE_NAME=${CUSTOM_IMAGE_NAME} \ - CUSTOM_IMAGE_VERSION=${CUSTOM_IMAGE_VERSION} + CUSTOM_BASE_IMAGE=${CUSTOM_BASE_IMAGE} diff --git a/scripts/install-chrome b/scripts/install-chrome index d8a5e59..a7135fb 100755 --- a/scripts/install-chrome +++ b/scripts/install-chrome @@ -3,27 +3,28 @@ set -xeuo pipefail IFS=$'\n\t' -if [[ $(dpkg --print-architecture) == arm64 ]]; then - echo "The arm64 does not have prebuilt chrome. Using chromium instead." +function build_debian() { + if [[ $(dpkg --print-architecture) == arm64 ]]; then + echo "The arm64 does not have prebuilt chrome. Using chromium instead." + apt-get update -q -y + apt-get install -y chromium chromium-driver + apt-get autoremove -yq + apt-get clean -yqq + rm -rf /var/lib/apt/lists/* + exit 0 + fi + + CHROME_VERSION=${1:-99.0.4844.74-1} + CHROME_DRIVER_VERSION=${2:-99.0.4844.51} + # We hard code the URL rather than using $CI_API_V4_URL $CI_PROJECT_ID, + # because we would need to forward those variables + CHROME_DOWNLOAD_URL_BASE="https://gitlab.com/api/v4/projects/1075790/packages/generic/google-chrome-stable" + export DEBIAN_FRONTEND=noninteractive + + curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - + echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list + apt-get update -q -y - apt-get install -y chromium chromium-driver - apt-get autoremove -yq - apt-get clean -yqq - rm -rf /var/lib/apt/lists/* - exit 0 -fi - -CHROME_VERSION=${1:-99.0.4844.74-1} -CHROME_DRIVER_VERSION=${2:-99.0.4844.51} -# We hard code the URL rather than using $CI_API_V4_URL $CI_PROJECT_ID, -# because we would need to forward those variables -CHROME_DOWNLOAD_URL_BASE="https://gitlab.com/api/v4/projects/1075790/packages/generic/google-chrome-stable" -export DEBIAN_FRONTEND=noninteractive - -curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - -echo "deb http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list - -apt-get update -q -y # Download from our package registry if we can't find the package in the apt repository echo "Searching for $CHROME_VERSION in apt repository" @@ -55,3 +56,12 @@ apt-get autoremove -yq apt-get clean -yqq rm -rf /var/lib/apt/lists/* rm -rf /etc/apt/sources.list.d/google*.list +} + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" +fi diff --git a/scripts/install-essentials b/scripts/install-essentials index 452387f..0db8a67 100755 --- a/scripts/install-essentials +++ b/scripts/install-essentials @@ -3,10 +3,6 @@ set -xeuo pipefail IFS=$'\n\t' -export DEBIAN_FRONTEND=noninteractive - -apt-get update - # We install `git-core` as some tooling expect `/usr/bin/git` # other tools that rely on PATH ordering will pick a one in `/usr/local` # if present @@ -47,30 +43,48 @@ function install_debian_bullseye_deps() { libre2-dev libevent-dev gettext rsync git-core lsb-release } -VERSION=`cat /etc/issue | cut -d ' ' -f 3` +function prepare_debian_environment() { + export DEBIAN_FRONTEND=noninteractive -case "$VERSION" in - 9) - install_debian_stretch_deps - ;; - 10) - install_debian_buster_deps - ;; - 11) - install_debian_bullseye_deps - ;; -esac + apt-get update -# Set UTF-8 -# http://stackoverflow.com/a/3182519/2137281 -LOC=$'LC_ALL=C.UTF-8\nLANG=C.UTF-8' -echo "$LOC" > /etc/environment -cat /etc/environment -echo "C.UTF-8 UTF-8" > /etc/locale.gen -locale-gen -dpkg-reconfigure locales -f noninteractive -p critical -locale -a + VERSION=`cat /etc/issue | cut -d ' ' -f 3` -apt-get autoremove -yq -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* + case "$VERSION" in + 9) + install_debian_stretch_deps + ;; + 10) + install_debian_buster_deps + ;; + 11) + install_debian_bullseye_deps + ;; + esac + + # Set UTF-8 + # http://stackoverflow.com/a/3182519/2137281 + LOC=$'LC_ALL=C.UTF-8\nLANG=C.UTF-8' + echo "$LOC" > /etc/environment + cat /etc/environment + echo "C.UTF-8 UTF-8" > /etc/locale.gen + locale-gen + dpkg-reconfigure locales -f noninteractive -p critical + locale -a + + apt-get autoremove -yq + apt-get clean -yqq + rm -rf /var/lib/apt/lists/* +} + +function prepare_ubi_environment() { + echo "UBI preparation scripts" +} + +if [[ $1 =~ debian ]]; then + export BUILD_OS=debian + prepare_debian_environment "$@" +elif [[ $1 =~ ubi ]]; then + export BUILD_OS=ubi + prepare_ubi_environment "$@" +fi diff --git a/scripts/install-gcloud b/scripts/install-gcloud index 5f55b68..687f6f7 100755 --- a/scripts/install-gcloud +++ b/scripts/install-gcloud @@ -2,24 +2,33 @@ set -xeuo pipefail -export DEBIAN_FRONTEND=noninteractive - GCLOUD_VERSION=${1} -apt-get update -apt-get -y install \ - apt-transport-https \ - python3 \ - gnupg +function build_debian() { + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get -y install \ + apt-transport-https \ + python3 \ + gnupg -curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - -echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + echo "deb https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list -apt-get update + apt-get update -PACKAGE_VERSION=$(apt-cache policy google-cloud-cli | awk -v dv=${GCLOUD_VERSION} '$1 ~ dv {print $1}') + PACKAGE_VERSION=$(apt-cache policy google-cloud-cli | awk -v dv=${GCLOUD_VERSION} '$1 ~ dv {print $1}') -apt-get install -y google-cloud-cli=${PACKAGE_VERSION} -apt-get -yq autoremove -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* + apt-get install -y google-cloud-cli=${PACKAGE_VERSION} + apt-get -yq autoremove + apt-get clean -yqq + rm -rf /var/lib/apt/lists/* +} + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" +fi diff --git a/scripts/install-noto-emoji b/scripts/install-noto-emoji index 6094939..d7c6f4f 100755 --- a/scripts/install-noto-emoji +++ b/scripts/install-noto-emoji @@ -1,42 +1,52 @@ #!/bin/bash # This script installs noto color emoji -apt-get update -apt-get install unzip +function build_debian() { + apt-get update + apt-get install unzip -cat > ~/.fonts.conf << EOM - - - - - - Noto Color Emoji - - true - - - - chrome - - - Noto Color Emoji - - - + cat > ~/.fonts.conf << EOM + + + + + + Noto Color Emoji + + true + + + + chrome + + + Noto Color Emoji + + + EOM -mkdir setup_fonts -cd setup_fonts -curl https://noto-website-2.storage.googleapis.com/pkgs/NotoColorEmoji-unhinted.zip -LO -unzip NotoColorEmoji-unhinted.zip + mkdir setup_fonts + cd setup_fonts + curl https://noto-website-2.storage.googleapis.com/pkgs/NotoColorEmoji-unhinted.zip -LO + unzip NotoColorEmoji-unhinted.zip -mkdir -p /usr/local/share/fonts -cp NotoColorEmoji.ttf /usr/local/share/fonts/ -ls -la /usr/local/share/fonts/ -chmod 644 /usr/local/share/fonts/NotoColorEmoji.ttf + mkdir -p /usr/local/share/fonts + cp NotoColorEmoji.ttf /usr/local/share/fonts/ + ls -la /usr/local/share/fonts/ + chmod 644 /usr/local/share/fonts/NotoColorEmoji.ttf -fc-cache -fv + fc-cache -fv -cd .. -rm -r setup_fonts -apt-get clean -yqq && rm -rf /var/lib/apt/lists/* + cd .. + rm -r setup_fonts + apt-get clean -yqq && rm -rf /var/lib/apt/lists/* +} + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" +fi diff --git a/scripts/install-postgresql b/scripts/install-postgresql index 2a11cb6..4c655c4 100755 --- a/scripts/install-postgresql +++ b/scripts/install-postgresql @@ -3,19 +3,30 @@ set -xeuo pipefail IFS=$'\n\t' POSTGRES_VERSION=${1:-12} -DEBIAN_VERSION=$(lsb_release -c -s) -export DEBIAN_FRONTEND=noninteractive +function build_debian() { + DEBIAN_VERSION=$(lsb_release -c -s) -# Uninstall the system client so that we don't have multiple versions -apt purge -y postgresql-client postgresql-client-common + export DEBIAN_FRONTEND=noninteractive -curl -sS -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - -echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DEBIAN_VERSION}-pgdg main" | tee /etc/apt/sources.list.d/postgresql.list + # Uninstall the system client so that we don't have multiple versions + apt purge -y postgresql-client postgresql-client-common -apt-get update -apt-get install -y postgresql-client-${POSTGRES_VERSION} + curl -sS -L https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DEBIAN_VERSION}-pgdg main" | tee /etc/apt/sources.list.d/postgresql.list -apt-get autoremove -yq -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* + apt-get update + apt-get install -y postgresql-client-${POSTGRES_VERSION} + + apt-get autoremove -yq + apt-get clean -yqq + rm -rf /var/lib/apt/lists/* +} + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" +fi diff --git a/scripts/install-ruby b/scripts/install-ruby index f152c5c..6489e63 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -16,69 +16,80 @@ JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${ BUNDLER_VERSION=${3:-""} RUBYGEMS_VERSION=${4:-""} -# Install needed packages -apt-get update -apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf -# Download jemalloc -mkdir -p /usr/src/jemalloc -cd /usr/src/jemalloc -curl --retry 6 -L -so jemalloc.tar.bz2 ${JEMALLOC_DOWNLOAD_URL} -echo "${JEMALLOC_DOWNLOAD_SHA256} jemalloc.tar.bz2" | sha256sum -c - +function build_debian() { + # Install needed packages + apt-get update + apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf -# Install jemalloc -tar -xjf jemalloc.tar.bz2 -rm jemalloc.tar.bz2 -cd jemalloc-${JEMALLOC_VERSION} -./autogen.sh --prefix=/usr --enable-prof -make -j "$(nproc)" install -cd /tmp + # Download jemalloc + mkdir -p /usr/src/jemalloc + cd /usr/src/jemalloc + curl --retry 6 -L -so jemalloc.tar.bz2 ${JEMALLOC_DOWNLOAD_URL} + echo "${JEMALLOC_DOWNLOAD_SHA256} jemalloc.tar.bz2" | sha256sum -c - -# Download Ruby -curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.gz -echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.gz" | sha256sum -c - + # Install jemalloc + tar -xjf jemalloc.tar.bz2 + rm jemalloc.tar.bz2 + cd jemalloc-${JEMALLOC_VERSION} + ./autogen.sh --prefix=/usr --enable-prof + make -j "$(nproc)" install + cd /tmp -# Skip installing Gem docs -mkdir -p /usr/local/etc -echo 'install: --no-document' >> /usr/local/etc/gemrc -echo 'update: --no-document' >> /usr/local/etc/gemrc + # Download Ruby + curl -fsSL "$RUBY_DOWNLOAD_URL" -o ruby.tar.gz + echo "${RUBY_DOWNLOAD_SHA256} ruby.tar.gz" | sha256sum -c - -# Unpack Ruby -mkdir -p /usr/src/ruby -tar -xzf ruby.tar.gz -C /usr/src/ruby --strip-components=1 -rm ruby.tar.gz -cd /usr/src/ruby + # Skip installing Gem docs + mkdir -p /usr/local/etc + echo 'install: --no-document' >> /usr/local/etc/gemrc + echo 'update: --no-document' >> /usr/local/etc/gemrc -# Apply patches -if [[ -d "/patches/ruby/$RUBY_VERSION" ]]; then - for i in "/patches/ruby/$RUBY_VERSION"/*.patch; do - echo "$i..." - patch -p1 -i "$i" - done + # Unpack Ruby + mkdir -p /usr/src/ruby + tar -xzf ruby.tar.gz -C /usr/src/ruby --strip-components=1 + rm ruby.tar.gz + cd /usr/src/ruby + + # Apply patches + if [[ -d "/patches/ruby/$RUBY_VERSION" ]]; then + for i in "/patches/ruby/$RUBY_VERSION"/*.patch; do + echo "$i..." + patch -p1 -i "$i" + done + fi + + # Compile + # This is needed for Ruby < 3.1 on Debian bullseye: https://bugs.ruby-lang.org/issues/18409 + export LDFLAGS="-Wl,--no-as-needed" + cflags="-fno-omit-frame-pointer" ./configure --enable-shared --with-jemalloc --disable-install-doc --disable-install-rdoc --disable-install-capi + make install -j $(nproc) + + # Install specific version of bundler if provided + if [[ "$BUNDLER_VERSION" != "" ]]; then gem install bundler -v $BUNDLER_VERSION; fi + + # Install specific version of RubyGems if provided + if [[ "$RUBYGEMS_VERSION" != "" ]]; then gem update --system $RUBYGEMS_VERSION; fi + + # Cleanup + cd / + rm -rf /usr/src/ruby /usr/src/jemalloc + apt-get purge -y --auto-remove ruby + + # Verify + # verify we have no "ruby" packages installed + ! dpkg -l | grep -i ruby + [ "$(command -v ruby)" = '/usr/local/bin/ruby' ] +} + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" fi -# Compile -# This is needed for Ruby < 3.1 on Debian bullseye: https://bugs.ruby-lang.org/issues/18409 -export LDFLAGS="-Wl,--no-as-needed" -cflags="-fno-omit-frame-pointer" ./configure --enable-shared --with-jemalloc --disable-install-doc --disable-install-rdoc --disable-install-capi -make install -j $(nproc) - -# Install specific version of bundler if provided -if [[ "$BUNDLER_VERSION" != "" ]]; then gem install bundler -v $BUNDLER_VERSION; fi - -# Install specific version of RubyGems if provided -if [[ "$RUBYGEMS_VERSION" != "" ]]; then gem update --system $RUBYGEMS_VERSION; fi - -# Cleanup -cd / -rm -rf /usr/src/ruby /usr/src/jemalloc -apt-get purge -y --auto-remove ruby - -# Verify -# verify we have no "ruby" packages installed -! dpkg -l | grep -i ruby -[ "$(command -v ruby)" = '/usr/local/bin/ruby' ] - # rough smoke test ruby --version gem --version diff --git a/scripts/install-www-gitlab-com b/scripts/install-www-gitlab-com index 6b851c2..66d02e3 100755 --- a/scripts/install-www-gitlab-com +++ b/scripts/install-www-gitlab-com @@ -3,55 +3,66 @@ set -xeuo pipefail IFS=$'\n\t' -export DEBIAN_FRONTEND=noninteractive +function build_debian() { + export DEBIAN_FRONTEND=noninteractive -# echo "deb http://deb.debian.org/debian testing main" | tee -a /etc/apt/sources.list.d/testing.list -# Install packages -apt-get update -apt-get install -yq --no-install-recommends \ - make gcc g++ locales \ - rsync git-core \ - ed file curl gnupg2 \ - unzip \ - python3 python3-pip python3-crcmod python-minimal \ - patch + # echo "deb http://deb.debian.org/debian testing main" | tee -a /etc/apt/sources.list.d/testing.list + # Install packages + apt-get update + apt-get install -yq --no-install-recommends \ + make gcc g++ locales \ + rsync git-core \ + ed file curl gnupg2 \ + unzip \ + python3 python3-pip python3-crcmod python-minimal \ + patch -# Install Imagemagick for cropping the pictures on the team page -apt-get install -yq --no-install-recommends imagemagick + # Install Imagemagick for cropping the pictures on the team page + apt-get install -yq --no-install-recommends imagemagick -# Install node & yarn -NODE_INSTALL_VERSION=16.14.2 -YARN_INSTALL_VERSION=1.22.10 -/scripts/install-node $NODE_INSTALL_VERSION $YARN_INSTALL_VERSION && node --version && yarn --version + # Install node & yarn + NODE_INSTALL_VERSION=16.14.2 + YARN_INSTALL_VERSION=1.22.10 + /scripts/install-node $NODE_INSTALL_VERSION $YARN_INSTALL_VERSION && node --version && yarn --version -# Install yamllint -# We need version 1.25.0+: https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst -YAMLLINT_VERSION=1.25.0 + # Install yamllint + # We need version 1.25.0+: https://github.com/adrienverge/yamllint/blob/master/CHANGELOG.rst + YAMLLINT_VERSION=1.25.0 -# Temporarily pin pyyaml and pathspec to reduce compatibility with packages from Debian bullseye (testing) -pip3 install yamllint==${YAMLLINT_VERSION} pyyaml==5.4.1 pathspec==0.8.1 + # Temporarily pin pyyaml and pathspec to reduce compatibility with packages from Debian bullseye (testing) + pip3 install yamllint==${YAMLLINT_VERSION} pyyaml==5.4.1 pathspec==0.8.1 -# Install gitlab-runner -curl -O -J -L https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-linux-amd64 -mv gitlab-ci-multi-runner-linux-amd64 /usr/bin/gitlab-runner-helper -chmod +x /usr/bin/gitlab-runner-helper + # Install gitlab-runner + curl -O -J -L https://gitlab-ci-multi-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-ci-multi-runner-linux-amd64 + mv gitlab-ci-multi-runner-linux-amd64 /usr/bin/gitlab-runner-helper + chmod +x /usr/bin/gitlab-runner-helper -# Patch gsutil to support gzip compression with rsync command: -# https://github.com/GoogleCloudPlatform/gsutil/pull/1430 -if [[ -d "/patches/gsutil" ]]; then - for i in /patches/gsutil/*.patch; do - echo "$i..." - patch -d /usr/lib/google-cloud-sdk/platform/gsutil -p1 -i "$i" - done + # Patch gsutil to support gzip compression with rsync command: + # https://github.com/GoogleCloudPlatform/gsutil/pull/1430 + if [[ -d "/patches/gsutil" ]]; then + for i in /patches/gsutil/*.patch; do + echo "$i..." + patch -d /usr/lib/google-cloud-sdk/platform/gsutil -p1 -i "$i" + done + fi + + # Set UTF-8 + echo "C.UTF-8 UTF-8" > /etc/locale.gen + locale-gen + update-locale LANG=C.UTF-8 LC_CTYPE=C.UTF-8 LC_ALL=C.UTF-8 + locale -a + + # Clean up + apt-get autoremove -yq + apt-get clean -yqq + rm -rf /var/lib/apt/lists/* +} +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" fi -# Set UTF-8 -echo "C.UTF-8 UTF-8" > /etc/locale.gen -locale-gen -update-locale LANG=C.UTF-8 LC_CTYPE=C.UTF-8 LC_ALL=C.UTF-8 -locale -a -# Clean up -apt-get autoremove -yq -apt-get clean -yqq -rm -rf /var/lib/apt/lists/* diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index 665251a..1c8292d 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -5,6 +5,14 @@ IFS=$'\n\t' SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" source "$SCRIPT_DIR/custom-docker.sh" +function get_base_image_reference() { + if [[ $1 =~ ^debian ]]; then + echo "$CUSTOM_DOCKER_ARCH/$1" + elif [[ $1 =~ ^ubi:8 ]]; then + echo "registry.access.redhat.com/ubi8/$1" + fi +} + function print_golang_args() { declare -A GOLANG_DOWNLOAD_SHA256 @@ -321,15 +329,12 @@ function parse_arguments() { *) echo "unknown architecture $(arch)"; exit 1;; esac - CUSTOM_IMAGE_NAME=debian - CUSTOM_IMAGE_VERSION=buster - for tool in "${PATH_TOOLS[@]}" "${TAG_TOOLS[@]}"; do if [ -n "${!tool}" ]; then version="${!tool}" case "$tool" in ARCH) CUSTOM_DOCKER_ARCH=$version ;; - DEBIAN) CUSTOM_IMAGE_VERSION=$version ;; + OS) CUSTOM_BASE_IMAGE=get_base_image_reference $version ;; RUBY) print_ruby_args $version ;; BUNDLER) print_bundler_args $version ;; RUBYGEMS) print_rubygems_args $version ;; @@ -353,10 +358,11 @@ function parse_arguments() { fi done - CUSTOM_IMAGE_NAME=$CUSTOM_DOCKER_ARCH/$CUSTOM_IMAGE_NAME # ex. https://hub.docker.com/r/amd64/debian/ + if [ -z "$CUSTOM_BASE_IMAGE" ]; then + CUSTOM_BASE_IMAGE="$CUSTOM_DOCKER_ARCH/debian:buster" + fi - printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" - printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$CUSTOM_IMAGE_VERSION" + printf -- "--build-arg CUSTOM_BASE_IMAGE=%s " "$CUSTOM_BASE_IMAGE" } function generate_command() { diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh index 4ecffbc..57ddc8e 100644 --- a/scripts/lib/custom-docker.sh +++ b/scripts/lib/custom-docker.sh @@ -1,12 +1,35 @@ -PATH_TOOLS=(DEBIAN RUBY GOLANG NODE POSTGRESQL) +# Note: Check out https://wiki.bash-hackers.org/syntax/pe for documentation on +# various variable operations used in this script. + +PATH_TOOLS=(OS RUBY GOLANG NODE POSTGRESQL) TAG_TOOLS=(BUNDLER RUBYGEMS GIT LFS CHROME YARN GRAPHICSMAGICK PGBOUNCER BAZELISK DOCKER BUILDX GCLOUD KUBECTL HELM) +# Generate the docker image path using the components that were specified via +# variables. +# For example, consider a CI job which specifies the following variables: +# OS: debian:bullseye +# RUBY: 2.7 +# GOLANG: 1.16 +# GIT: 2.33 +# PGBOUNCER: 1.14 +# POSTGRESQL: 11 +# With the above variables, this function will return +# `debian-bullseye-ruby-2.7-golang-1.16-postgresql-11` function get_image_path() { local path path="" for tool in "${PATH_TOOLS[@]}"; do if [[ -n "${!tool}" ]]; then - path="${path}-${tool,,}-${!tool}" + if [[ "${tool}" == "OS" ]]; then + # The OS variable's value is following : + # format. We split that string into individual components. + distro=${!tool%:*} + version=${!tool#*:} + path="${path}-${distro}-${version}" + else + # Convert the tool name into lowercase using `,,` operator + path="${path}-${tool,,}-${!tool}" + fi fi done @@ -17,15 +40,26 @@ function get_image_path() { fi } +# Generate the image tag using the components that were specified via variables. +# For example, consider a CI job which specifies the following variables: +# OS: debian:bullseye +# RUBY: 2.7 +# GOLANG: 1.16 +# GIT: 2.33 +# PGBOUNCER: 1.14 +# POSTGRESQL: 11 +# For that job, this function will return +# `git-2.33-pgbouncer-1.14` function get_image_tag() { local tag tag="" for tool in "${TAG_TOOLS[@]}"; do if [[ -n "${!tool}" ]]; then + # Convert the tool name into lowercase using `,,` operator tag="${tag}-${tool,,}-${!tool}" fi done - + if [[ -n "$tag" ]]; then echo "${tag:1}" else