Keep the build images DRY

.gitlab-ci.yml

@@ -27,20 +27,41 @@ before_script:
   only:
     - master
 
-ruby-2.1-git-2.7-phantomjs-2.1 test: *test_build
+.test_custom: &test_custom
-ruby-2.3-git-2.7-phantomjs-2.1 test: *test_build
+  stage: test
-ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1 test: *test_build
+  script:
-ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1 test: *test_build
+    # Hack to set an array in /bin/sh
-ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_build
+    # http://unix.stackexchange.com/a/137571
-ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6 test: *test_build
+    - set -- $CI_JOB_NAME
-ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_build
+    - ./scripts/custom-docker-build $1
+  except:
+    - master
+
+.build_and_deploy_custom: &build_and_deploy_custom
+  stage: build
+  script:
+    - ./scripts/custom-docker-build $CI_BUILD_NAME -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
+    - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
+  only:
+    - master
+
+# Tests
+
+ruby-2.1-git-2.7-phantomjs-2.1 test: *test_custom
+ruby-2.3-git-2.7-phantomjs-2.1 test: *test_custom
+ruby-2.3-golang-1.8-git-2.13 test: *test_custom
+ruby-2.3-golang-1.8-git-2.8 test: *test_custom
+ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1 test: *test_custom
+ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1 test: *test_custom
+ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_custom
+ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6 test: *test_custom
+ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_custom
+golang-1.8-git-2.8.4 test: *test_custom
+golang-1.8-git-2.13 test: *test_custom
+
 www-gitlab-com test: *test_build
 doc-gitlab-com test: *test_build
 gitlab-qa test: *test_build
-golang-1.5-git-2.8.4 test: *test_build
-golang-1.6-git-2.8.4 test: *test_build
-golang-1.7-git-2.8.4 test: *test_build
-golang-1.8-git-2.8.4 test: *test_build
 omnibus-gitlab-centos6 test: *test_build
 omnibus-gitlab-centos7 test: *test_build
 omnibus-gitlab-jessie test: *test_build
@@ -50,20 +71,23 @@ omnibus-gitlab-trusty test: *test_build
 omnibus-gitlab-wheezy test: *test_build
 omnibus-gitlab-xenial test: *test_build
 
-ruby-2.1-git-2.7-phantomjs-2.1: *build_and_deploy
+# Builds
-ruby-2.3-git-2.7-phantomjs-2.1: *build_and_deploy
+
-ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy
+ruby-2.1-git-2.7-phantomjs-2.1: *build_and_deploy_custom
-ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy
+ruby-2.3-git-2.7-phantomjs-2.1: *build_and_deploy_custom
-ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy
+ruby-2.3-golang-1.8-git-2.13: *build_and_deploy_custom
-ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy
+ruby-2.3-golang-1.8-git-2.8: *build_and_deploy_custom
-ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy
+ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy_custom
+ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy_custom
+ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy_custom
+ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy_custom
+ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy_custom
+golang-1.8-git-2.8.4: *build_and_deploy_custom
+golang-1.8-git-2.13: *build_and_deploy_custom
+
 www-gitlab-com: *build_and_deploy
 doc-gitlab-com: *build_and_deploy
 gitlab-qa: *build_and_deploy
-golang-1.5-git-2.8.4: *build_and_deploy
-golang-1.6-git-2.8.4: *build_and_deploy
-golang-1.7-git-2.8.4: *build_and_deploy
-golang-1.8-git-2.8.4: *build_and_deploy
 omnibus-gitlab-centos6: *build_and_deploy
 omnibus-gitlab-centos7: *build_and_deploy
 omnibus-gitlab-jessie: *build_and_deploy
@@ -72,3 +96,4 @@ omnibus-gitlab-opensuse42.2: *build_and_deploy
 omnibus-gitlab-trusty: *build_and_deploy
 omnibus-gitlab-wheezy: *build_and_deploy
 omnibus-gitlab-xenial: *build_and_deploy
+

Dockerfile.custom

@@ -0,0 +1,49 @@
+#
+# The arguments in this file are configured in
+# /scripts/custom-docker-build
+#
+
+ARG CUSTOM_IMAGE_NAME
+ARG CUSTOM_IMAGE_VERSION
+FROM ${CUSTOM_IMAGE_NAME}:${CUSTOM_IMAGE_VERSION}
+
+ADD / /
+
+RUN /scripts/install-essentials
+
+ENV PATH $PATH:/usr/local/go/bin
+
+# Git
+ARG GIT_VERSION
+ARG GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.gz
+ARG GIT_DOWNLOAD_SHA256
+RUN if [ -n "$GIT_VERSION" ]; then /scripts/install-git && git --version; fi
+
+# Chrome
+ARG CHROME_VERSION
+RUN if [ -n "$CHROME_VERSION" ]; then /scripts/install-chrome $CHROME_VERSION; fi
+
+# PhantomJS
+ARG PHANTOMJS_VERSION
+RUN if [ -n "$PHANTOMJS_VERSION" ] ; then /scripts/install-phantomjs $PHANTOMJS_VERSION; fi
+
+# NodeJS
+ARG NODE_VERSION
+RUN if [ -n "$NODE_VERSION" ] ; then /scripts/install-node $NODE_VERSION && node --version; fi
+
+# Golang
+ARG GOLANG_VERSION
+ARG GOLANG_DOWNLOAD_SHA256
+RUN if [ -n "$GOLANG_VERSION" ] ; then /scripts/install-golang && go version; fi
+
+# Postgres
+# Codedebt: currently this will only install the default version of postgres,
+# not the stipulated version
+ARG POSTGRES_VERSION
+RUN if [ -n "$POSTGRES_VERSION" ] ; then /scripts/install-postgresql; fi
+
+
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8

Dockerfile.golang-1.5-git-2.8.4

@@ -1,16 +0,0 @@
-FROM golang:1.5
-
-ENV GIT_VERSION 2.8.4
-ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
-ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
-
-# Install Build-deps
-RUN apt-get update && apt-get install -y \
-    dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
-    gettext libz-dev libssl-dev
-
-ADD . $PWD
-
-RUN scripts/install-git
-
-ENV PATH /usr/local/bin:$PATH

Dockerfile.golang-1.6-git-2.8.4

@@ -1,16 +0,0 @@
-FROM golang:1.6
-
-ENV GIT_VERSION 2.8.4
-ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
-ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
-
-# Install Build-deps
-RUN apt-get update && apt-get install -y \
-    dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
-    gettext libz-dev libssl-dev
-
-ADD . $PWD
-
-RUN scripts/install-git
-
-ENV PATH /usr/local/bin:$PATH

Dockerfile.golang-1.7-git-2.8.4

@@ -1,16 +0,0 @@
-FROM golang:1.7
-
-ENV GIT_VERSION 2.8.4
-ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
-ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
-
-# Install Build-deps
-RUN apt-get update && apt-get install -y \
-    dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
-    gettext libz-dev libssl-dev
-
-ADD . $PWD
-
-RUN scripts/install-git
-
-ENV PATH /usr/local/bin:$PATH

Dockerfile.golang-1.8-git-2.8.4

@@ -1,16 +0,0 @@
-FROM golang:1.8
-
-ENV GIT_VERSION 2.8.4
-ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
-ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
-
-# Install Build-deps
-RUN apt-get update && apt-get install -y \
-    dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
-    gettext libz-dev libssl-dev
-
-ADD . $PWD
-
-RUN scripts/install-git
-
-ENV PATH /usr/local/bin:$PATH

Dockerfile.ruby-2.1-git-2.7-phantomjs-2.1

@@ -1,10 +0,0 @@
-FROM ruby:2.1
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-
-RUN locale-gen en_US.UTF-8  
-ENV LANG en_US.UTF-8  
-ENV LANGUAGE en_US:en  
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.3-git-2.7-phantomjs-2.1

@@ -1,10 +0,0 @@
-FROM ruby:2.3
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1

@@ -1,11 +0,0 @@
-FROM ruby:2.3.3
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-RUN /scripts/install-node
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6

@@ -1,15 +0,0 @@
-FROM ruby:2.3.3
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-chrome
-RUN /scripts/install-node
-RUN /scripts/install-golang
-RUN /scripts/install-postgresql
-
-ENV PATH /usr/local/go/bin:$PATH
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1

@@ -1,14 +0,0 @@
-FROM ruby:2.3.3
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-RUN /scripts/install-node
-RUN /scripts/install-golang
-
-ENV PATH /usr/local/go/bin:$PATH
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6

@@ -1,15 +0,0 @@
-FROM ruby:2.3.3
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-RUN /scripts/install-node
-RUN /scripts/install-golang
-RUN /scripts/install-postgresql
-
-ENV PATH /usr/local/go/bin:$PATH
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

Dockerfile.ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6

@@ -1,15 +0,0 @@
-FROM ruby:2.4.1
-
-ADD / /
-RUN /scripts/install-essentials
-RUN /scripts/install-phantomjs 2.1.1
-RUN /scripts/install-node
-RUN /scripts/install-golang
-RUN /scripts/install-postgresql
-
-ENV PATH /usr/local/go/bin:$PATH
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8

README.md

@@ -4,3 +4,58 @@ various parts of GitLab:
 1. Build Omnibus packages
 1. Test GitLab-CE/EE project
 1. Build gitlab-org/www-gitlab-com project
+
+# Adding a new image
+
+There are two methods to adding a new image:
+1. Use a docker buildfile
+2. Use a custom image with versioned features
+
+## Use a docker buildfile
+
+A bespoke image is one that uses it's own Dockerfile. In the interests of
+avoiding repetition, it's better to use the custom docker build if possible.
+
+Assuming the image you want to add is called `new-image`....
+
+1. Add a test task: `new-image test: *test_build`
+1. Add a new build task: `new-image: *build_and_deploy`
+1. Add a Dockerfile: `Dockerfile.new-image` 
+
+
+## Use a custom image with versioned features
+
+To add a new image to the build, edit [`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
+
+With custom images, the name of the image is used to define the contents (as opposed to use a bespoke Dockerfile build).
+
+For this reason, the name needs to be correctly defined. 
+
+The name is defined as `[feature]-[version]` pairs, separated by `-`.
+
+Here are some example build names:
+
+1. `ruby-2.1-git-2.7-phantomjs-2.1`
+1. `ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6`
+1. `ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6`
+
+The first pair represents the base image and version. So `ruby-2.1-...` will use the `ruby:2.1` base image, while
+`golang-1.8` will use the `golang:1.8` base image.
+
+Each of the following parts represents another feature and version. Available 
+options are:
+
+1. `golang`
+2. `git`
+3. `chrome`
+4. `node`
+5. `phantomjs`
+6. `postgres`
+
+### Adding a new build
+
+As an example, if you want to add new image for Ruby 2.4 with `git` 2.14 and `golang` 1.9, 
+the name would be `ruby-2.4-golang-1.9-git-2.14`. 
+
+1. Add a test task: `ruby-2.4-golang-1.9-git-2.14 test: *test_custom`
+1. Add a new build task: `ruby-2.4-golang-1.9-git-2.14: *build_and_deploy_custom`

scripts/custom-docker-build

@@ -0,0 +1,105 @@
+#!/bin/sh
+
+set -e
+IFS=$'\n\t'
+
+function print_golang_args() {
+    GOLANG_VERSION=$1
+
+    case "$GOLANG_VERSION" in
+        1.8)
+            GOLANG_DOWNLOAD_SHA256=253ab94104ee3923e228a2cb2116e5e462ad3ebaeea06ff04463479d7f12d27ca
+            ;;
+        *) echo "Unknown golang version $1"; exit 1;
+    esac
+
+    printf -- "--build-arg GOLANG_VERSION=%s " "$GOLANG_VERSION"
+    printf -- "--build-arg GOLANG_DOWNLOAD_SHA256=%s " "$GOLANG_DOWNLOAD_SHA256"
+}
+
+function print_chrome_args() {
+    case "$1" in
+        60|60.0) CHROME_VERSION=60.0.3112.90-1 ;;
+        *) echo "Unknown chrome version $1"; exit 1;
+    esac
+    printf -- "--build-arg CHROME_VERSION=%s " "$CHROME_VERSION"
+}
+
+function print_git_args() {
+    case "$1" in
+        2.7)
+            # Code-debt: When the build refers to "git 2.7" its actually "default" git.
+            # so don't actually install a git 2.7 as a default copy is already installed
+            return
+            ;;
+        2.8|2.8.4)
+            GIT_VERSION=2.8.4
+            GIT_DOWNLOAD_SHA256=626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
+            ;;
+        2.13)
+            GIT_VERSION=2.13.0
+            GIT_DOWNLOAD_SHA256=9f2fa8040ebafc0c2caae4a9e2cb385c6f16c0525bcb0fbd84938bc796372e80
+            ;;
+        *) echo "Unknown git version $1"; exit 1;
+    esac
+
+    printf -- "--build-arg GIT_VERSION=%s " "$GIT_VERSION"
+    printf -- "--build-arg GIT_DOWNLOAD_SHA256=%s " "$GIT_DOWNLOAD_SHA256"
+}
+
+function print_phantomjs_args() {
+    case "$1" in
+        2.1) PHANTOMJS_VERSION=2.1.1 ;;
+        *) echo "Unknown phantomjs version $1"; exit 1;
+    esac
+    printf -- "--build-arg PHANTOMJS_VERSION=%s " "$PHANTOMJS_VERSION"
+}
+
+function print_node_args() {
+    case "$1" in
+        7.1) NODE_VERSION=7.x ;;
+        *) echo "Unknown chrome version $1"; exit 1;
+    esac
+    printf -- "--build-arg NODE_VERSION=%s " "$NODE_VERSION"
+}
+
+function print_postgres_args() {
+    printf -- "--build-arg POSTGRES_VERSION=%s " "$1"
+}
+
+function parse_arguments() {
+    read base
+    read base_version
+    printf -- "-f Dockerfile.custom " "$base"
+    printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$base"
+    printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$base_version"
+    while read image; do
+        read version
+        case "$image" in
+            golang) print_golang_args $version ;;
+            chrome) print_chrome_args $version ;;
+            git) print_git_args $version ;;
+            phantomjs) print_phantomjs_args $version ;;
+            node) print_node_args $version ;;
+            postgresql) print_postgres_args $version ;;
+            *) exit 1;;
+        esac
+    done
+}
+
+function generate_command() {
+    buildimage_name=$1; shift;
+
+    printf -- "docker build "
+    echo $buildimage_name | tr '-' '\n' | parse_arguments
+
+    for i in "$@"
+    do
+        printf -- "%s " "$i"
+    done
+    printf -- ".\\n"
+}
+
+docker_command=$(generate_command $@)
+echo "$1: executing $docker_command"
+eval $docker_command

scripts/install-essentials

@@ -11,7 +11,7 @@ apt-get install -y \
 	libxslt-dev libcurl4-openssl-dev libicu-dev \
 	logrotate python-docutils pkg-config cmake nodejs \
 	libkrb5-dev postgresql-client mysql-client unzip \
-	libre2-dev \
+	libre2-dev gettext \
 
 echo deb http://httpredir.debian.org/debian stretch main > /etc/apt/sources.list.d/stretch.list
 apt-get update -yqqq

scripts/install-node

@@ -2,8 +2,10 @@
 
 set -xeo pipefail
 
+NODE_VERSION=${1:-7.x}
+
 # add official debian repos for node and yarn
-curl -sS -L https://deb.nodesource.com/setup_7.x | bash -
+curl -sS -L https://deb.nodesource.com/setup_${NODE_VERSION} | bash -
 curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list