From 690bb34f74ecfb77f5dc39b57042d23e84971989 Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Tue, 5 Jul 2022 12:01:56 +0530 Subject: [PATCH 1/6] Fix build environment detection Signed-off-by: Balasankar "Balu" C --- Dockerfile.custom | 11 ++++------- scripts/install-essentials | 2 -- scripts/lib/custom-docker-build | 30 ++++++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 11 deletions(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 9adc458..0a7924d 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -6,14 +6,12 @@ ARG CUSTOM_BASE_IMAGE FROM ${CUSTOM_BASE_IMAGE} -# We are setting this ARG again because it is required in install-essentials -# script. ARG defined before FROM can't be used afterwards. -# Check https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact -ARG CUSTOM_BASE_IMAGE +ARG BUILD_OS +ARG BUILD_ARCH ADD / / -RUN /scripts/install-essentials ${CUSTOM_BASE_IMAGE} +RUN /scripts/install-essentials ${BUILD_OS} ENV PATH $PATH:/usr/local/go/bin @@ -123,5 +121,4 @@ ENV RUBY_VERSION=${RUBY_VERSION} \ DOCKER_VERSION=${DOCKER_VERSION} \ BAZELISK_VERSION=${BAZELISK_VERSION} \ GCLOUD_VERSION=${GCLOUD_VERSION} \ - KUBECTL_VERSION=${KUBECTL_VERSION} \ - CUSTOM_BASE_IMAGE=${CUSTOM_BASE_IMAGE} + KUBECTL_VERSION=${KUBECTL_VERSION} diff --git a/scripts/install-essentials b/scripts/install-essentials index 0db8a67..038cc9d 100755 --- a/scripts/install-essentials +++ b/scripts/install-essentials @@ -82,9 +82,7 @@ function prepare_ubi_environment() { } if [[ $1 =~ debian ]]; then - export BUILD_OS=debian prepare_debian_environment "$@" elif [[ $1 =~ ubi ]]; then - export BUILD_OS=ubi prepare_ubi_environment "$@" fi diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build index a5056d6..52c9cb6 100755 --- a/scripts/lib/custom-docker-build +++ b/scripts/lib/custom-docker-build @@ -301,10 +301,32 @@ function parse_arguments() { # armhf | armhf | armhf | arm32v7 # aarch64 | aarch64 | arm64 | arm64v8 + # We make use of 4 variables related to the build environment. + # 1. `CUSTOM_DOCKER_IMAGE` - Defines the reference of image used as + # base for building images. Follows a + # valid docker image reference format + # - "arm64v8/debian:buster" + # - "registry.access.redhat.com/ubi8/ubi:8.6" + # 2. `CUSTOM_DOCKER_ARCH` - Defines the architecture of the docker + # image. Follows docker architecture + # format - `amd64`/`arm32v7`/`arm64v8`. + # 3. `BUILD_OS` - Used by installation scripts of various + # components to detect the OS being worked + # on. Follows regular docker image:tag + # format - "debian:buster" or "ubi:8.6". + # 4. `BUILD_ARCH` - Used by installation scripts of various + # components to detect the architecture + # being worked on. Follows general + # architecture names - `amd64`/`arm64`. + # + # The first two variables are used to pick the docker image used as + # base for builds. The last two variables are used to detect OS and + # architecture while building components. + # defaults case $(arch) in - x86_64) CUSTOM_DOCKER_ARCH=amd64 ;; - aarch64) CUSTOM_DOCKER_ARCH=arm64v8 ;; + x86_64) CUSTOM_DOCKER_ARCH=amd64; BUILD_ARCH=amd64 ;; + aarch64) CUSTOM_DOCKER_ARCH=arm64v8; BUILD_ARCH=arm64 ;; *) echo "unknown architecture $(arch)"; exit 1;; esac @@ -340,7 +362,11 @@ function parse_arguments() { CUSTOM_BASE_IMAGE="$CUSTOM_DOCKER_ARCH/debian:buster" fi + BUILD_OS=${OS:-"debian:buster"} + printf -- "--build-arg CUSTOM_BASE_IMAGE=%s " "$CUSTOM_BASE_IMAGE" + printf -- "--build-arg BUILD_OS=%s " "$BUILD_OS" + printf -- "--build-arg BUILD_ARCH=%s " "$BUILD_ARCH" } function generate_command() { From 9441b216087c7d234df85b99c9fddd28c960204e Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Mon, 4 Jul 2022 19:18:21 +0530 Subject: [PATCH 2/6] Install essential packages in UBI Signed-off-by: Balasankar "Balu" C --- scripts/install-essentials | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/install-essentials b/scripts/install-essentials index 038cc9d..e25b84e 100755 --- a/scripts/install-essentials +++ b/scripts/install-essentials @@ -78,7 +78,17 @@ function prepare_debian_environment() { } function prepare_ubi_environment() { - echo "UBI preparation scripts" + yum update -y + + yum install -by --nodocs \ + autoconf cmake gcc gcc-c++ make patch perl bzip2 \ + libedit ncurses uuid libarchive curl-devel \ + libicu-devel libffi-devel libuuid-devel openssl-devel \ + ncurses-devel pcre2-devel zlib-devel libstdc++-static \ + libevent-devel redhat-lsb-core procps-ng + + yum autoremove -y + yum clean -y all } if [[ $1 =~ debian ]]; then From 0149acb910485a4d5418f84ba3d1e62fc941c62e Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Mon, 4 Jul 2022 19:18:35 +0530 Subject: [PATCH 3/6] Support building Ruby in UBI Signed-off-by: Balasankar "Balu" C --- scripts/install-ruby | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/scripts/install-ruby b/scripts/install-ruby index 6489e63..87be7d5 100755 --- a/scripts/install-ruby +++ b/scripts/install-ruby @@ -16,12 +16,7 @@ JEMALLOC_DOWNLOAD_URL="https://github.com/jemalloc/jemalloc/releases/download/${ BUNDLER_VERSION=${3:-""} RUBYGEMS_VERSION=${4:-""} - -function build_debian() { - # Install needed packages - apt-get update - apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf - +function build_common() { # Download jemalloc mkdir -p /usr/src/jemalloc cd /usr/src/jemalloc @@ -74,6 +69,15 @@ function build_debian() { # Cleanup cd / rm -rf /usr/src/ruby /usr/src/jemalloc +} + +function build_debian() { + # Install needed packages + apt-get update + apt-get install -y --no-install-recommends bison dpkg-dev libgdbm-dev autoconf + + build_common + apt-get purge -y --auto-remove ruby # Verify @@ -82,6 +86,21 @@ function build_debian() { [ "$(command -v ruby)" = '/usr/local/bin/ruby' ] } +function build_ubi() { + yum update -y + + build_common + + yum remove -y ruby + yum autoremove -y + yum clean -y all + + # Verify + # verify we have no "ruby" packages installed + ! yum list installed | grep -i ruby + [ "$(command -v ruby)" = '/usr/local/bin/ruby' ] +} + BUILD_OS=${BUILD_OS:-debian} if [[ $BUILD_OS =~ debian ]]; then From 475f5af39009db92466d0a01a38c3a5e0933b08f Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Tue, 5 Jul 2022 11:37:29 +0530 Subject: [PATCH 4/6] Support building Golang-FIPS in UBI Signed-off-by: Balasankar "Balu" C --- scripts/install-golang | 45 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/scripts/install-golang b/scripts/install-golang index 0ce4e4a..ce543a5 100755 --- a/scripts/install-golang +++ b/scripts/install-golang @@ -5,9 +5,44 @@ set -xeou pipefail INSTALL_GOLANG_VERSION=${1} GOLANG_DOWNLOAD_SHA256=${2} -GOLANG_DOWNLOAD_URL="https://golang.org/dl/go${INSTALL_GOLANG_VERSION}.linux-$(dpkg --print-architecture).tar.gz" +GOLANG_DOWNLOAD_URL="https://golang.org/dl/go${INSTALL_GOLANG_VERSION}.linux-${BUILD_ARCH}.tar.gz" -curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz -echo "${GOLANG_DOWNLOAD_SHA256} golang.tar.gz" | sha256sum -c - -tar -C /usr/local -xzf golang.tar.gz -rm golang.tar.gz +function build_debian() { + curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz + echo "${GOLANG_DOWNLOAD_SHA256} golang.tar.gz" | sha256sum -c - + tar -C /usr/local -xzf golang.tar.gz + rm golang.tar.gz +} + +function build_ubi() { + GO_MAJOR_VERSION=${INSTALL_GOLANG_VERSION%.*} + GOLANG_FIPS_BRANCH="go${GO_MAJOR_VERSION}-openssl-fips" + + mkdir -p /tmp/golang + curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz + echo "${GOLANG_DOWNLOAD_SHA256} golang.tar.gz" | sha256sum -c - + tar -C /tmp/golang -xzf golang.tar.gz + + # For UBI, we will be installing golang-fips + git clone https://github.com/golang-fips/go.git --branch ${GOLANG_FIPS_BRANCH} --single-branch --depth 1 /usr/local/go + + cd /usr/local/go/src + + PATH=$PATH:/tmp/golang/go/bin CGO_ENABLED=1 ./make.bash + + rm -rf /usr/local/go/pkg/*/cmd /usr/local/go/pkg/bootstrap \ + /usr/local/go/pkg/obj /usr/local/go/pkg/tool/*/api \ + /usr/local/go/pkg/tool/*/go_bootstrap /usr/local/go/src/cmd/dist/dist \ + /usr/local/go/.git* /tmp/golang + + ln -sf /usr/local/go/bin/go /usr/local/go/bin/gofmt /usr/local/go/bin/godoc /usr/local/bin/ +} + + +BUILD_OS=${BUILD_OS:-debian} + +if [[ $BUILD_OS =~ debian ]]; then + build_debian "$@" +elif [[ $BUILD_OS =~ ubi ]]; then + build_ubi "$@" +fi From a19885da576cb62415e236169b009b67a6cb46c2 Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Tue, 5 Jul 2022 15:24:53 +0530 Subject: [PATCH 5/6] Make locale setting OS-dependent Signed-off-by: Balasankar "Balu" C --- Dockerfile.custom | 2 +- scripts/generate-locale | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100755 scripts/generate-locale diff --git a/Dockerfile.custom b/Dockerfile.custom index 0a7924d..ace788d 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -100,7 +100,7 @@ ARG HELM_DOWNLOAD_SHA256 RUN if [ -n "$HELM_VERSION" ] ; then /scripts/install-helm $HELM_VERSION $HELM_DOWNLOAD_SHA256; fi -RUN locale-gen C.UTF-8 +RUN /scripts/generate-locale ENV LANG=C.UTF-8 \ LANGUAGE=C \ LC_ALL=C.UTF-8 diff --git a/scripts/generate-locale b/scripts/generate-locale new file mode 100755 index 0000000..8ee70be --- /dev/null +++ b/scripts/generate-locale @@ -0,0 +1,10 @@ +#!/bin/bash + +set -xeou pipefail + +BUILD_OS=${BUILD_OS:-debian} + +# For UBI, the locale is already present in the image. +if [[ $BUILD_OS =~ debian ]]; then + locale-gen C.UTF-8 +fi From 67adad0f40c6a5323755b77a5546d4831b7f4b0b Mon Sep 17 00:00:00 2001 From: "Balasankar \"Balu\" C" Date: Tue, 5 Jul 2022 10:43:13 +0530 Subject: [PATCH 6/6] Add UBI variants of Gitaly images Signed-off-by: Balasankar "Balu" C --- .gitlab/ci/gitaly.images.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/ci/gitaly.images.yml b/.gitlab/ci/gitaly.images.yml index 315f338..6ae5f84 100644 --- a/.gitlab/ci/gitaly.images.yml +++ b/.gitlab/ci/gitaly.images.yml @@ -10,3 +10,7 @@ gitaly: RUBY: ['2.7', '3.0'] GOLANG: ['1.17', '1.18'] GIT: ['2.36'] + - OS: ['ubi:8.6'] + RUBY: ['2.7'] + GOLANG: ['1.17'] + GIT: ['2.36']