Merge branch 'use-arguments' into 'master'

Keep the build images DRY

See merge request !52
This commit is contained in:
Stan Hu 2017-08-08 17:58:52 +00:00
commit 08cda4d0d9
17 changed files with 260 additions and 178 deletions

View file

@ -27,20 +27,41 @@ before_script:
only:
- master
ruby-2.1-git-2.7-phantomjs-2.1 test: *test_build
ruby-2.3-git-2.7-phantomjs-2.1 test: *test_build
ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1 test: *test_build
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1 test: *test_build
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_build
ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6 test: *test_build
ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_build
.test_custom: &test_custom
stage: test
script:
# Hack to set an array in /bin/sh
# http://unix.stackexchange.com/a/137571
- set -- $CI_JOB_NAME
- ./scripts/custom-docker-build $1
except:
- master
.build_and_deploy_custom: &build_and_deploy_custom
stage: build
script:
- ./scripts/custom-docker-build $CI_BUILD_NAME -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
- docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
only:
- master
# Tests
ruby-2.1-git-2.7-phantomjs-2.1 test: *test_custom
ruby-2.3-git-2.7-phantomjs-2.1 test: *test_custom
ruby-2.3-golang-1.8-git-2.13 test: *test_custom
ruby-2.3-golang-1.8-git-2.8 test: *test_custom
ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1 test: *test_custom
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1 test: *test_custom
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_custom
ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6 test: *test_custom
ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6 test: *test_custom
golang-1.8-git-2.8.4 test: *test_custom
golang-1.8-git-2.13 test: *test_custom
www-gitlab-com test: *test_build
doc-gitlab-com test: *test_build
gitlab-qa test: *test_build
golang-1.5-git-2.8.4 test: *test_build
golang-1.6-git-2.8.4 test: *test_build
golang-1.7-git-2.8.4 test: *test_build
golang-1.8-git-2.8.4 test: *test_build
omnibus-gitlab-centos6 test: *test_build
omnibus-gitlab-centos7 test: *test_build
omnibus-gitlab-jessie test: *test_build
@ -50,20 +71,23 @@ omnibus-gitlab-trusty test: *test_build
omnibus-gitlab-wheezy test: *test_build
omnibus-gitlab-xenial test: *test_build
ruby-2.1-git-2.7-phantomjs-2.1: *build_and_deploy
ruby-2.3-git-2.7-phantomjs-2.1: *build_and_deploy
ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy
ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy
ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy
# Builds
ruby-2.1-git-2.7-phantomjs-2.1: *build_and_deploy_custom
ruby-2.3-git-2.7-phantomjs-2.1: *build_and_deploy_custom
ruby-2.3-golang-1.8-git-2.13: *build_and_deploy_custom
ruby-2.3-golang-1.8-git-2.8: *build_and_deploy_custom
ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy_custom
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1: *build_and_deploy_custom
ruby-2.3.3-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy_custom
ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6: *build_and_deploy_custom
ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6: *build_and_deploy_custom
golang-1.8-git-2.8.4: *build_and_deploy_custom
golang-1.8-git-2.13: *build_and_deploy_custom
www-gitlab-com: *build_and_deploy
doc-gitlab-com: *build_and_deploy
gitlab-qa: *build_and_deploy
golang-1.5-git-2.8.4: *build_and_deploy
golang-1.6-git-2.8.4: *build_and_deploy
golang-1.7-git-2.8.4: *build_and_deploy
golang-1.8-git-2.8.4: *build_and_deploy
omnibus-gitlab-centos6: *build_and_deploy
omnibus-gitlab-centos7: *build_and_deploy
omnibus-gitlab-jessie: *build_and_deploy
@ -72,3 +96,4 @@ omnibus-gitlab-opensuse42.2: *build_and_deploy
omnibus-gitlab-trusty: *build_and_deploy
omnibus-gitlab-wheezy: *build_and_deploy
omnibus-gitlab-xenial: *build_and_deploy

49
Dockerfile.custom Normal file
View file

@ -0,0 +1,49 @@
#
# The arguments in this file are configured in
# /scripts/custom-docker-build
#
ARG CUSTOM_IMAGE_NAME
ARG CUSTOM_IMAGE_VERSION
FROM ${CUSTOM_IMAGE_NAME}:${CUSTOM_IMAGE_VERSION}
ADD / /
RUN /scripts/install-essentials
ENV PATH $PATH:/usr/local/go/bin
# Git
ARG GIT_VERSION
ARG GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git/git-${GIT_VERSION}.tar.gz
ARG GIT_DOWNLOAD_SHA256
RUN if [ -n "$GIT_VERSION" ]; then /scripts/install-git && git --version; fi
# Chrome
ARG CHROME_VERSION
RUN if [ -n "$CHROME_VERSION" ]; then /scripts/install-chrome $CHROME_VERSION; fi
# PhantomJS
ARG PHANTOMJS_VERSION
RUN if [ -n "$PHANTOMJS_VERSION" ] ; then /scripts/install-phantomjs $PHANTOMJS_VERSION; fi
# NodeJS
ARG NODE_VERSION
RUN if [ -n "$NODE_VERSION" ] ; then /scripts/install-node $NODE_VERSION && node --version; fi
# Golang
ARG GOLANG_VERSION
ARG GOLANG_DOWNLOAD_SHA256
RUN if [ -n "$GOLANG_VERSION" ] ; then /scripts/install-golang && go version; fi
# Postgres
# Codedebt: currently this will only install the default version of postgres,
# not the stipulated version
ARG POSTGRES_VERSION
RUN if [ -n "$POSTGRES_VERSION" ] ; then /scripts/install-postgresql; fi
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,16 +0,0 @@
FROM golang:1.5
ENV GIT_VERSION 2.8.4
ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
# Install Build-deps
RUN apt-get update && apt-get install -y \
dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
gettext libz-dev libssl-dev
ADD . $PWD
RUN scripts/install-git
ENV PATH /usr/local/bin:$PATH

View file

@ -1,16 +0,0 @@
FROM golang:1.6
ENV GIT_VERSION 2.8.4
ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
# Install Build-deps
RUN apt-get update && apt-get install -y \
dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
gettext libz-dev libssl-dev
ADD . $PWD
RUN scripts/install-git
ENV PATH /usr/local/bin:$PATH

View file

@ -1,16 +0,0 @@
FROM golang:1.7
ENV GIT_VERSION 2.8.4
ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
# Install Build-deps
RUN apt-get update && apt-get install -y \
dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
gettext libz-dev libssl-dev
ADD . $PWD
RUN scripts/install-git
ENV PATH /usr/local/bin:$PATH

View file

@ -1,16 +0,0 @@
FROM golang:1.8
ENV GIT_VERSION 2.8.4
ENV GIT_DOWNLOAD_URL https://www.kernel.org/pub/software/scm/git/git-$GIT_VERSION.tar.gz
ENV GIT_DOWNLOAD_SHA256 626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
# Install Build-deps
RUN apt-get update && apt-get install -y \
dh-autoreconf libcurl4-gnutls-dev libexpat1-dev \
gettext libz-dev libssl-dev
ADD . $PWD
RUN scripts/install-git
ENV PATH /usr/local/bin:$PATH

View file

@ -1,10 +0,0 @@
FROM ruby:2.1
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,10 +0,0 @@
FROM ruby:2.3
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,11 +0,0 @@
FROM ruby:2.3.3
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN /scripts/install-node
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,15 +0,0 @@
FROM ruby:2.3.3
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-chrome
RUN /scripts/install-node
RUN /scripts/install-golang
RUN /scripts/install-postgresql
ENV PATH /usr/local/go/bin:$PATH
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,14 +0,0 @@
FROM ruby:2.3.3
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN /scripts/install-node
RUN /scripts/install-golang
ENV PATH /usr/local/go/bin:$PATH
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,15 +0,0 @@
FROM ruby:2.3.3
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN /scripts/install-node
RUN /scripts/install-golang
RUN /scripts/install-postgresql
ENV PATH /usr/local/go/bin:$PATH
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -1,15 +0,0 @@
FROM ruby:2.4.1
ADD / /
RUN /scripts/install-essentials
RUN /scripts/install-phantomjs 2.1.1
RUN /scripts/install-node
RUN /scripts/install-golang
RUN /scripts/install-postgresql
ENV PATH /usr/local/go/bin:$PATH
RUN locale-gen en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8

View file

@ -4,3 +4,58 @@ various parts of GitLab:
1. Build Omnibus packages
1. Test GitLab-CE/EE project
1. Build gitlab-org/www-gitlab-com project
# Adding a new image
There are two methods to adding a new image:
1. Use a docker buildfile
2. Use a custom image with versioned features
## Use a docker buildfile
A bespoke image is one that uses it's own Dockerfile. In the interests of
avoiding repetition, it's better to use the custom docker build if possible.
Assuming the image you want to add is called `new-image`....
1. Add a test task: `new-image test: *test_build`
1. Add a new build task: `new-image: *build_and_deploy`
1. Add a Dockerfile: `Dockerfile.new-image`
## Use a custom image with versioned features
To add a new image to the build, edit [`.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab-build-images/blob/master/.gitlab-ci.yml).
With custom images, the name of the image is used to define the contents (as opposed to use a bespoke Dockerfile build).
For this reason, the name needs to be correctly defined.
The name is defined as `[feature]-[version]` pairs, separated by `-`.
Here are some example build names:
1. `ruby-2.1-git-2.7-phantomjs-2.1`
1. `ruby-2.3.3-golang-1.8-git-2.7-chrome-60.0-node-7.1-postgresql-9.6`
1. `ruby-2.4.1-golang-1.8-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6`
The first pair represents the base image and version. So `ruby-2.1-...` will use the `ruby:2.1` base image, while
`golang-1.8` will use the `golang:1.8` base image.
Each of the following parts represents another feature and version. Available
options are:
1. `golang`
2. `git`
3. `chrome`
4. `node`
5. `phantomjs`
6. `postgres`
### Adding a new build
As an example, if you want to add new image for Ruby 2.4 with `git` 2.14 and `golang` 1.9,
the name would be `ruby-2.4-golang-1.9-git-2.14`.
1. Add a test task: `ruby-2.4-golang-1.9-git-2.14 test: *test_custom`
1. Add a new build task: `ruby-2.4-golang-1.9-git-2.14: *build_and_deploy_custom`

105
scripts/custom-docker-build Executable file
View file

@ -0,0 +1,105 @@
#!/bin/sh
set -e
IFS=$'\n\t'
function print_golang_args() {
GOLANG_VERSION=$1
case "$GOLANG_VERSION" in
1.8)
GOLANG_DOWNLOAD_SHA256=253ab94104ee3923e228a2cb2116e5e462ad3ebaeea06ff04463479d7f12d27ca
;;
*) echo "Unknown golang version $1"; exit 1;
esac
printf -- "--build-arg GOLANG_VERSION=%s " "$GOLANG_VERSION"
printf -- "--build-arg GOLANG_DOWNLOAD_SHA256=%s " "$GOLANG_DOWNLOAD_SHA256"
}
function print_chrome_args() {
case "$1" in
60|60.0) CHROME_VERSION=60.0.3112.90-1 ;;
*) echo "Unknown chrome version $1"; exit 1;
esac
printf -- "--build-arg CHROME_VERSION=%s " "$CHROME_VERSION"
}
function print_git_args() {
case "$1" in
2.7)
# Code-debt: When the build refers to "git 2.7" its actually "default" git.
# so don't actually install a git 2.7 as a default copy is already installed
return
;;
2.8|2.8.4)
GIT_VERSION=2.8.4
GIT_DOWNLOAD_SHA256=626e319f8a24fc0866167ea5f6bf3e2f38f69d6cb2e59e150f13709ca3ebf301
;;
2.13)
GIT_VERSION=2.13.0
GIT_DOWNLOAD_SHA256=9f2fa8040ebafc0c2caae4a9e2cb385c6f16c0525bcb0fbd84938bc796372e80
;;
*) echo "Unknown git version $1"; exit 1;
esac
printf -- "--build-arg GIT_VERSION=%s " "$GIT_VERSION"
printf -- "--build-arg GIT_DOWNLOAD_SHA256=%s " "$GIT_DOWNLOAD_SHA256"
}
function print_phantomjs_args() {
case "$1" in
2.1) PHANTOMJS_VERSION=2.1.1 ;;
*) echo "Unknown phantomjs version $1"; exit 1;
esac
printf -- "--build-arg PHANTOMJS_VERSION=%s " "$PHANTOMJS_VERSION"
}
function print_node_args() {
case "$1" in
7.1) NODE_VERSION=7.x ;;
*) echo "Unknown chrome version $1"; exit 1;
esac
printf -- "--build-arg NODE_VERSION=%s " "$NODE_VERSION"
}
function print_postgres_args() {
printf -- "--build-arg POSTGRES_VERSION=%s " "$1"
}
function parse_arguments() {
read base
read base_version
printf -- "-f Dockerfile.custom " "$base"
printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$base"
printf -- "--build-arg CUSTOM_IMAGE_VERSION=%s " "$base_version"
while read image; do
read version
case "$image" in
golang) print_golang_args $version ;;
chrome) print_chrome_args $version ;;
git) print_git_args $version ;;
phantomjs) print_phantomjs_args $version ;;
node) print_node_args $version ;;
postgresql) print_postgres_args $version ;;
*) exit 1;;
esac
done
}
function generate_command() {
buildimage_name=$1; shift;
printf -- "docker build "
echo $buildimage_name | tr '-' '\n' | parse_arguments
for i in "$@"
do
printf -- "%s " "$i"
done
printf -- ".\\n"
}
docker_command=$(generate_command $@)
echo "$1: executing $docker_command"
eval $docker_command

View file

@ -11,7 +11,7 @@ apt-get install -y \
libxslt-dev libcurl4-openssl-dev libicu-dev \
logrotate python-docutils pkg-config cmake nodejs \
libkrb5-dev postgresql-client mysql-client unzip \
libre2-dev \
libre2-dev gettext \
echo deb http://httpredir.debian.org/debian stretch main > /etc/apt/sources.list.d/stretch.list
apt-get update -yqqq

View file

@ -2,8 +2,10 @@
set -xeo pipefail
NODE_VERSION=${1:-7.x}
# add official debian repos for node and yarn
curl -sS -L https://deb.nodesource.com/setup_7.x | bash -
curl -sS -L https://deb.nodesource.com/setup_${NODE_VERSION} | bash -
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list