Welcome to the matrix

This refactors our custom image building to utilize GitLab parallel jobs
with a matrix.

This makes it easier to parse what kind of matrix we are going to build
in our docker images. Furthermore instead of splitting the image name,
we can simply pull the versions of the tools from the environment
variables.

The ultimative reason: Dogfooding
This commit is contained in:
Lukas Eipert 2021-11-26 17:06:29 +01:00
parent 32d1908296
commit a9cf88c469
11 changed files with 325 additions and 292 deletions

View file

@ -1,6 +1,7 @@
# we support merge request workflow only
include:
- template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml'
- local: '.gitlab/ci/*.yml'
default:
image: docker:20.10.9-git
@ -8,7 +9,6 @@ default:
- docker:20.10.9-dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
- source scripts/build-helpers.sh
- apk add -U bash
tags:
- gitlab-org-docker
@ -22,273 +22,6 @@ variables:
DOCKER_DRIVER: overlay2
DOCKER_HOST: tcp://docker:2375
# Run test job always
.test:
stage: test
rules:
- when: always
# Run build job only on a default branch when triggered manually
.deploy:
stage: deploy
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual
.test_build: &test_build
extends: .test
script:
# Hack to set an array in /bin/sh
# http://unix.stackexchange.com/a/137571
- set -- $CI_JOB_NAME
- docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true
- docker build --cache-from="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" .
.build_and_deploy: &build_and_deploy
extends: .deploy
script:
# Hack to set an array in /bin/sh
# http://unix.stackexchange.com/a/137571
- set -- $CI_JOB_NAME
- docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true
- docker build --cache-from="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" .
- docker push "$CI_REGISTRY_IMAGE:$1"
.test_custom: &test_custom
extends: .test
script:
# Hack to set an array in /bin/sh
# http://unix.stackexchange.com/a/137571
- set -- $CI_JOB_NAME
- ./scripts/custom-docker-build "$1"
rules:
- when: always
.build_and_deploy_custom: &build_and_deploy_custom
extends: .deploy
script:
# Hack to set an array in /bin/sh
# http://unix.stackexchange.com/a/137571
- set -- $CI_JOB_NAME
- ./scripts/custom-docker-build "$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1"
- docker push "$CI_REGISTRY_IMAGE:$1"
# Tests
# Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml
ruby-2.7-golang-1.16-git-2.31 test: *test_custom
ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom
ruby-2.7-golang-1.17-git-2.31 test: *test_custom
ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom
ruby-3.0-golang-1.16-git-2.31 test: *test_custom
ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom
ruby-3.0-golang-1.17-git-2.31 test: *test_custom
ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom
ruby-2.7-golang-1.16-git-2.33 test: *test_custom
ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom
ruby-2.7-golang-1.17-git-2.33 test: *test_custom
ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom
ruby-3.0-golang-1.16-git-2.33 test: *test_custom
ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom
ruby-3.0-golang-1.17-git-2.33 test: *test_custom
ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom
# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom
# Used by GitLab/KAS integration tests
ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom
# Used by GitLab's compile-production-assets and compile-test-assets jobs
ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom
ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom
ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom
# Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml
ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom
ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom
ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom
# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml
ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom
ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom
ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom
ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom
# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml
alpine-helm test: *test_build
danger test: *test_build
danger-ruby-2.6.6 test: *test_build
gitlab-charts-build-base test: *test_build
gitlab-charts-build-base-helm-3 test: *test_build
gitlab-helm3.5-kubectl1.17 test: *test_build
gitlab-qa-ruby-2.7 test: *test_build
gitlab-qa-ruby-3.0 test: *test_build
gitlab-qa-alpine-ruby-2.7 test: *test_build
gitlab-qa-alpine-ruby-3.0 test: *test_build
gitlab-puppeteer test: *test_build
kaniko test: *test_build
omnibus-gitlab-bionic test: *test_build
omnibus-gitlab-centos7 test: *test_build
omnibus-gitlab-centos8 test: *test_build
omnibus-gitlab-jessie test: *test_build
omnibus-gitlab-opensuse15.1 test: *test_build
omnibus-gitlab-opensuse15.2 test: *test_build
omnibus-gitlab-opensuse15.3 test: *test_build
omnibus-gitlab-stretch test: *test_build
omnibus-gitlab-buster test: *test_build
omnibus-gitlab-bullseye test: *test_build
omnibus-gitlab-xenial test: *test_build
omnibus-gitlab-focal test: *test_build
omnibus-gitlab-depscan test: *test_build
omnibus-gitlab-cve-search test: *test_build
release-tools test: *test_build
sitespeed-gitlab test: *test_build
ubi-release test: *test_build
www-gitlab-com-3.0 test: *test_build
build-git: *test_build
# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
terraform test: *test_build
# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting
golangci-lint-alpine test: *test_build
# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator
gitlab-operator-build-base test: *test_build
alpine-bash-jq-curl-git test: *test_build
# Used by frontend related tasks for linting GraphQL
apollo test: *test_build
# Builds
# Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml
ruby-2.7-golang-1.16-git-2.31 push: *build_and_deploy_custom
ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-2.7-golang-1.17-git-2.31 push: *build_and_deploy_custom
ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-3.0-golang-1.16-git-2.31 push: *build_and_deploy_custom
ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-3.0-golang-1.17-git-2.31 push: *build_and_deploy_custom
ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-2.7-golang-1.16-git-2.33 push: *build_and_deploy_custom
ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-2.7-golang-1.17-git-2.33 push: *build_and_deploy_custom
ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-3.0-golang-1.16-git-2.33 push: *build_and_deploy_custom
ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom
ruby-3.0-golang-1.17-git-2.33 push: *build_and_deploy_custom
ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom
# Used by GitLab CE/EE: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/.gitlab-ci.yml
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom
# Used by GitLab/KAS integration tests
ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
ruby-2.7.2.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom
# Used by GitLab's compile-production-assets and compile-test-assets jobs
ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom
ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom
# Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml
ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom
ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom
ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom
# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml
ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom
ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom
ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom
ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom
# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml
alpine-helm push: *build_and_deploy
danger push: *build_and_deploy
danger-ruby-2.6.6 push: *build_and_deploy
gitlab-charts-build-base push: *build_and_deploy
gitlab-charts-build-base-helm-3 push: *build_and_deploy
gitlab-helm3.5-kubectl1.17 push: *build_and_deploy
gitlab-qa-ruby-2.7 push: *build_and_deploy
gitlab-qa-ruby-3.0 push: *build_and_deploy
gitlab-qa-alpine-ruby-2.7 push: *build_and_deploy
gitlab-qa-alpine-ruby-3.0 push: *build_and_deploy
gitlab-puppeteer push: *build_and_deploy
kaniko push: *build_and_deploy
omnibus-gitlab-bionic push: *build_and_deploy
omnibus-gitlab-centos7 push: *build_and_deploy
omnibus-gitlab-centos8 push: *build_and_deploy
omnibus-gitlab-jessie push: *build_and_deploy
omnibus-gitlab-opensuse15.1 push: *build_and_deploy
omnibus-gitlab-opensuse15.2 push: *build_and_deploy
omnibus-gitlab-opensuse15.3 push: *build_and_deploy
omnibus-gitlab-stretch push: *build_and_deploy
omnibus-gitlab-buster push: *build_and_deploy
omnibus-gitlab-bullseye push: *build_and_deploy
omnibus-gitlab-xenial push: *build_and_deploy
omnibus-gitlab-focal push: *build_and_deploy
omnibus-gitlab-depscan push: *build_and_deploy
omnibus-gitlab-cve-search push: *build_and_deploy
release-tools push: *build_and_deploy
sitespeed-gitlab push: *build_and_deploy
ubi-release push: *build_and_deploy
www-gitlab-com-3.0 push: *build_and_deploy
build-git push: *build_and_deploy
# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
terraform push: *build_and_deploy
# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting
golangci-lint-alpine push: *build_and_deploy
# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator
gitlab-operator-build-base push: *build_and_deploy
# Used by frontend related tasks for linting GraphQL
apollo push: *build_and_deploy
alpine-bash-jq-curl-git push: *build_and_deploy
cache-google-chrome:
stage: automation
rules: