From a9cf88c469c9183d05b4ed1c908c990ad671346b Mon Sep 17 00:00:00 2001 From: Lukas Eipert Date: Fri, 26 Nov 2021 17:06:29 +0100 Subject: [PATCH] Welcome to the matrix This refactors our custom image building to utilize GitLab parallel jobs with a matrix. This makes it easier to parse what kind of matrix we are going to build in our docker images. Furthermore instead of splitting the image name, we can simply pull the versions of the tools from the environment variables. The ultimative reason: Dogfooding --- .gitlab-ci.yml | 269 +------------------------------- .gitlab/ci/custom.images.yml | 108 +++++++++++++ .gitlab/ci/customers.images.yml | 45 ++++++ .gitlab/ci/definitions.yml | 25 +++ .gitlab/ci/gitaly.images.yml | 19 +++ .gitlab/ci/gitlab.images.yml | 46 ++++++ .gitlab/ci/kas.images.yml | 24 +++ scripts/build-helpers.sh | 0 scripts/custom-docker-build | 51 +++--- scripts/custom-docker-push | 18 +++ scripts/lib/custom-docker.sh | 12 ++ 11 files changed, 325 insertions(+), 292 deletions(-) create mode 100644 .gitlab/ci/custom.images.yml create mode 100644 .gitlab/ci/customers.images.yml create mode 100644 .gitlab/ci/definitions.yml create mode 100644 .gitlab/ci/gitaly.images.yml create mode 100644 .gitlab/ci/gitlab.images.yml create mode 100644 .gitlab/ci/kas.images.yml delete mode 100644 scripts/build-helpers.sh create mode 100755 scripts/custom-docker-push create mode 100644 scripts/lib/custom-docker.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65658e7..b5e47b9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ # we support merge request workflow only include: - template: 'Workflows/MergeRequest-Pipelines.gitlab-ci.yml' + - local: '.gitlab/ci/*.yml' default: image: docker:20.10.9-git @@ -8,7 +9,6 @@ default: - docker:20.10.9-dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY" - - source scripts/build-helpers.sh - apk add -U bash tags: - gitlab-org-docker @@ -22,273 +22,6 @@ variables: DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 -# Run test job always -.test: - stage: test - rules: - - when: always - -# Run build job only on a default branch when triggered manually -.deploy: - stage: deploy - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: manual - -.test_build: &test_build - extends: .test - script: - # Hack to set an array in /bin/sh - # http://unix.stackexchange.com/a/137571 - - set -- $CI_JOB_NAME - - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true - - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . - -.build_and_deploy: &build_and_deploy - extends: .deploy - script: - # Hack to set an array in /bin/sh - # http://unix.stackexchange.com/a/137571 - - set -- $CI_JOB_NAME - - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true - - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . - - docker push "$CI_REGISTRY_IMAGE:$1" - -.test_custom: &test_custom - extends: .test - script: - # Hack to set an array in /bin/sh - # http://unix.stackexchange.com/a/137571 - - set -- $CI_JOB_NAME - - ./scripts/custom-docker-build "$1" - rules: - - when: always - -.build_and_deploy_custom: &build_and_deploy_custom - extends: .deploy - script: - # Hack to set an array in /bin/sh - # http://unix.stackexchange.com/a/137571 - - set -- $CI_JOB_NAME - - ./scripts/custom-docker-build "$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" - - docker push "$CI_REGISTRY_IMAGE:$1" - -# Tests - -# Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml -ruby-2.7-golang-1.16-git-2.31 test: *test_custom -ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom -ruby-2.7-golang-1.17-git-2.31 test: *test_custom -ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom -ruby-3.0-golang-1.16-git-2.31 test: *test_custom -ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 test: *test_custom -ruby-3.0-golang-1.17-git-2.31 test: *test_custom -ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 test: *test_custom -ruby-2.7-golang-1.16-git-2.33 test: *test_custom -ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom -ruby-2.7-golang-1.17-git-2.33 test: *test_custom -ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom -ruby-3.0-golang-1.16-git-2.33 test: *test_custom -ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 test: *test_custom -ruby-3.0-golang-1.17-git-2.33 test: *test_custom -ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 test: *test_custom - - -# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 test: *test_custom - - -# Used by GitLab/KAS integration tests -ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 test: *test_custom - -# Used by GitLab's compile-production-assets and compile-test-assets jobs -ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom -ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom -ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 test: *test_custom - -# Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml -ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom -ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom -ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom -ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom -ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom -ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1: *test_custom -ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom -ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5: *test_custom - -# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml -ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom -ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom -ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom -ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1: *test_custom - -# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml -alpine-helm test: *test_build - -danger test: *test_build -danger-ruby-2.6.6 test: *test_build -gitlab-charts-build-base test: *test_build -gitlab-charts-build-base-helm-3 test: *test_build -gitlab-helm3.5-kubectl1.17 test: *test_build -gitlab-qa-ruby-2.7 test: *test_build -gitlab-qa-ruby-3.0 test: *test_build -gitlab-qa-alpine-ruby-2.7 test: *test_build -gitlab-qa-alpine-ruby-3.0 test: *test_build -gitlab-puppeteer test: *test_build -kaniko test: *test_build -omnibus-gitlab-bionic test: *test_build -omnibus-gitlab-centos7 test: *test_build -omnibus-gitlab-centos8 test: *test_build -omnibus-gitlab-jessie test: *test_build -omnibus-gitlab-opensuse15.1 test: *test_build -omnibus-gitlab-opensuse15.2 test: *test_build -omnibus-gitlab-opensuse15.3 test: *test_build -omnibus-gitlab-stretch test: *test_build -omnibus-gitlab-buster test: *test_build -omnibus-gitlab-bullseye test: *test_build -omnibus-gitlab-xenial test: *test_build -omnibus-gitlab-focal test: *test_build -omnibus-gitlab-depscan test: *test_build -omnibus-gitlab-cve-search test: *test_build - -release-tools test: *test_build -sitespeed-gitlab test: *test_build -ubi-release test: *test_build -www-gitlab-com-3.0 test: *test_build -build-git: *test_build -# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml -terraform test: *test_build -# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting -golangci-lint-alpine test: *test_build -# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator -gitlab-operator-build-base test: *test_build - -alpine-bash-jq-curl-git test: *test_build - -# Used by frontend related tasks for linting GraphQL -apollo test: *test_build - -# Builds - -# Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml -ruby-2.7-golang-1.16-git-2.31 push: *build_and_deploy_custom -ruby-2.7-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-2.7-golang-1.17-git-2.31 push: *build_and_deploy_custom -ruby-2.7-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-3.0-golang-1.16-git-2.31 push: *build_and_deploy_custom -ruby-3.0-golang-1.16-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-3.0-golang-1.17-git-2.31 push: *build_and_deploy_custom -ruby-3.0-golang-1.17-git-2.31-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-2.7-golang-1.16-git-2.33 push: *build_and_deploy_custom -ruby-2.7-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-2.7-golang-1.17-git-2.33 push: *build_and_deploy_custom -ruby-2.7-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-3.0-golang-1.16-git-2.33 push: *build_and_deploy_custom -ruby-3.0-golang-1.16-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom -ruby-3.0-golang-1.17-git-2.33 push: *build_and_deploy_custom -ruby-3.0-golang-1.17-git-2.33-pgbouncer-1.14 push: *build_and_deploy_custom - -# Used by GitLab CE/EE: https://gitlab.com/gitlab-org/gitlab-ee/blob/master/.gitlab-ci.yml -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-11-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-chrome-89-node-14.15-yarn-1.22-postgresql-13-graphicsmagick-1.3.36 push: *build_and_deploy_custom - -# Used by GitLab/KAS integration tests -ruby-2.7.2.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.31-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom -ruby-2.7.2.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom -ruby-2.7.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom -ruby-3.0.patched-golang-1.16-git-2.33-lfs-2.9-node-14.15-yarn-1.22-postgresql-12-graphicsmagick-1.3.36-bazelisk-1.9.0 push: *build_and_deploy_custom - -# Used by GitLab's compile-production-assets and compile-test-assets jobs -ruby-2.7-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0-git-2.31-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-2.7-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom -ruby-3.0-git-2.33-lfs-2.9-node-14.15-yarn-1.22-graphicsmagick-1.3.36 push: *build_and_deploy_custom - -# Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml -ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -ruby-2.6.6-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom -ruby-2.7.2-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -ruby-2.7-git-2.31-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -ruby-2.6.6-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5-docker-19.03.1 push: *build_and_deploy_custom -ruby-2.7.2-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom -ruby-2.7-git-2.33-lfs-2.9-chrome-89-node-12.22-yarn-1.22-postgresql-9.5 push: *build_and_deploy_custom - -# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml -ruby-2.7.2-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom -ruby-2.7-git-2.31-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom -ruby-2.7.2-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom -ruby-2.7-git-2.33-chrome-89-node-12.22-yarn-1.22-docker-19.03.1 push: *build_and_deploy_custom - -# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml -alpine-helm push: *build_and_deploy - -danger push: *build_and_deploy -danger-ruby-2.6.6 push: *build_and_deploy -gitlab-charts-build-base push: *build_and_deploy -gitlab-charts-build-base-helm-3 push: *build_and_deploy -gitlab-helm3.5-kubectl1.17 push: *build_and_deploy -gitlab-qa-ruby-2.7 push: *build_and_deploy -gitlab-qa-ruby-3.0 push: *build_and_deploy -gitlab-qa-alpine-ruby-2.7 push: *build_and_deploy -gitlab-qa-alpine-ruby-3.0 push: *build_and_deploy -gitlab-puppeteer push: *build_and_deploy -kaniko push: *build_and_deploy -omnibus-gitlab-bionic push: *build_and_deploy -omnibus-gitlab-centos7 push: *build_and_deploy -omnibus-gitlab-centos8 push: *build_and_deploy -omnibus-gitlab-jessie push: *build_and_deploy -omnibus-gitlab-opensuse15.1 push: *build_and_deploy -omnibus-gitlab-opensuse15.2 push: *build_and_deploy -omnibus-gitlab-opensuse15.3 push: *build_and_deploy -omnibus-gitlab-stretch push: *build_and_deploy -omnibus-gitlab-buster push: *build_and_deploy -omnibus-gitlab-bullseye push: *build_and_deploy -omnibus-gitlab-xenial push: *build_and_deploy -omnibus-gitlab-focal push: *build_and_deploy -omnibus-gitlab-depscan push: *build_and_deploy -omnibus-gitlab-cve-search push: *build_and_deploy -release-tools push: *build_and_deploy -sitespeed-gitlab push: *build_and_deploy -ubi-release push: *build_and_deploy -www-gitlab-com-3.0 push: *build_and_deploy -build-git push: *build_and_deploy -# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml -terraform push: *build_and_deploy -# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting -golangci-lint-alpine push: *build_and_deploy -# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator -gitlab-operator-build-base push: *build_and_deploy -# Used by frontend related tasks for linting GraphQL -apollo push: *build_and_deploy - -alpine-bash-jq-curl-git push: *build_and_deploy cache-google-chrome: stage: automation rules: diff --git a/.gitlab/ci/custom.images.yml b/.gitlab/ci/custom.images.yml new file mode 100644 index 0000000..f3b7f49 --- /dev/null +++ b/.gitlab/ci/custom.images.yml @@ -0,0 +1,108 @@ +.test_build: &test_build + extends: .test + script: + # Hack to set an array in /bin/sh + # http://unix.stackexchange.com/a/137571 + - set -- $CI_JOB_NAME + - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true + - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . + +.build_and_deploy: &build_and_deploy + extends: .deploy + script: + # Hack to set an array in /bin/sh + # http://unix.stackexchange.com/a/137571 + - set -- $CI_JOB_NAME + - docker pull --quiet "$CI_REGISTRY_IMAGE:$1" || true + - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" . + - docker push "$CI_REGISTRY_IMAGE:$1" + +# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml +alpine-helm test: *test_build + +danger test: *test_build +danger-ruby-2.6.6 test: *test_build +gitlab-charts-build-base test: *test_build +gitlab-charts-build-base-helm-3 test: *test_build +gitlab-helm3.5-kubectl1.17 test: *test_build +gitlab-qa-ruby-2.7 test: *test_build +gitlab-qa-ruby-3.0 test: *test_build +gitlab-qa-alpine-ruby-2.7 test: *test_build +gitlab-qa-alpine-ruby-3.0 test: *test_build +gitlab-puppeteer test: *test_build +kaniko test: *test_build +omnibus-gitlab-bionic test: *test_build +omnibus-gitlab-centos7 test: *test_build +omnibus-gitlab-centos8 test: *test_build +omnibus-gitlab-jessie test: *test_build +omnibus-gitlab-opensuse15.1 test: *test_build +omnibus-gitlab-opensuse15.2 test: *test_build +omnibus-gitlab-opensuse15.3 test: *test_build +omnibus-gitlab-stretch test: *test_build +omnibus-gitlab-buster test: *test_build +omnibus-gitlab-xenial test: *test_build +omnibus-gitlab-focal test: *test_build +omnibus-gitlab-depscan test: *test_build +omnibus-gitlab-cve-search test: *test_build + +release-tools test: *test_build +sitespeed-gitlab test: *test_build +ubi-release test: *test_build +www-gitlab-com-3.0 test: *test_build +build-git: *test_build +# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +terraform test: *test_build +# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting +golangci-lint-alpine test: *test_build +# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator +gitlab-operator-build-base test: *test_build + +alpine-bash-jq-curl-git test: *test_build + +# Used by frontend related tasks for linting GraphQL +apollo test: *test_build + + +# Used by https://gitlab.com/gitlab-org/charts/auto-deploy-app/-/blob/master/.gitlab-ci.yml +alpine-helm push: *build_and_deploy + +danger push: *build_and_deploy +danger-ruby-2.6.6 push: *build_and_deploy +gitlab-charts-build-base push: *build_and_deploy +gitlab-charts-build-base-helm-3 push: *build_and_deploy +gitlab-helm3.5-kubectl1.17 push: *build_and_deploy +gitlab-qa-ruby-2.7 push: *build_and_deploy +gitlab-qa-ruby-3.0 push: *build_and_deploy +gitlab-qa-alpine-ruby-2.7 push: *build_and_deploy +gitlab-qa-alpine-ruby-3.0 push: *build_and_deploy +gitlab-puppeteer push: *build_and_deploy +kaniko push: *build_and_deploy +omnibus-gitlab-bionic push: *build_and_deploy +omnibus-gitlab-centos7 push: *build_and_deploy +omnibus-gitlab-centos8 push: *build_and_deploy +omnibus-gitlab-jessie push: *build_and_deploy +omnibus-gitlab-opensuse15.1 push: *build_and_deploy +omnibus-gitlab-opensuse15.2 push: *build_and_deploy +omnibus-gitlab-opensuse15.3 push: *build_and_deploy +omnibus-gitlab-stretch push: *build_and_deploy +omnibus-gitlab-buster push: *build_and_deploy +omnibus-gitlab-xenial push: *build_and_deploy +omnibus-gitlab-focal push: *build_and_deploy +omnibus-gitlab-depscan push: *build_and_deploy +omnibus-gitlab-cve-search push: *build_and_deploy +release-tools push: *build_and_deploy +sitespeed-gitlab push: *build_and_deploy +ubi-release push: *build_and_deploy +www-gitlab-com-3.0 push: *build_and_deploy +build-git push: *build_and_deploy +# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +terraform push: *build_and_deploy +# Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting +golangci-lint-alpine push: *build_and_deploy +# Used by gitlab-operator: https://gitlab.com/gitlab-org/gl-openshift/gitlab-operator +gitlab-operator-build-base push: *build_and_deploy + +alpine-bash-jq-curl-git push: *build_and_deploy + +# Used by frontend related tasks for linting GraphQL +apollo push: *build_and_deploy diff --git a/.gitlab/ci/customers.images.yml b/.gitlab/ci/customers.images.yml new file mode 100644 index 0000000..d290d65 --- /dev/null +++ b/.gitlab/ci/customers.images.yml @@ -0,0 +1,45 @@ +# Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml +.customers-matrix: + variables: + LFS: '2.9' + CHROME: '89' + NODE: '12.22' + YARN: '1.22' + POSTGRESQL: '9.5' + parallel: + matrix: + - RUBY: ['2.6.6', '2.7', '2.7.2'] + GIT: ['2.31', '2.33'] + DOCKER: ['19.03.1', ''] + +customers test: + extends: + - .customers-matrix + - .test_custom + +customers push: + extends: + - .customers-matrix + - .build_and_deploy_custom + +# Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml +.customers-qa-matrix: + variables: + CHROME: '89' + NODE: '12.22' + YARN: '1.22' + DOCKER: '19.03.1' + parallel: + matrix: + - RUBY: ['2.7', '2.7.2'] + GIT: ['2.31', '2.33'] + +customers-qa-assets test: + extends: + - .customers-qa-matrix + - .test_custom + +customers-qa-assets push: + extends: + - .customers-qa-matrix + - .build_and_deploy_custom diff --git a/.gitlab/ci/definitions.yml b/.gitlab/ci/definitions.yml new file mode 100644 index 0000000..4beea63 --- /dev/null +++ b/.gitlab/ci/definitions.yml @@ -0,0 +1,25 @@ +# Run test job always +.test: + stage: test + rules: + - when: always + +# Run build job only on a default branch when triggered manually +.deploy: + stage: deploy + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + when: manual + +.test_custom: + extends: .test + script: + - ./scripts/custom-docker-build + rules: + - when: always + +.build_and_deploy_custom: + extends: .deploy + script: + - ./scripts/custom-docker-build --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" + - ./scripts/custom-docker-push diff --git a/.gitlab/ci/gitaly.images.yml b/.gitlab/ci/gitaly.images.yml new file mode 100644 index 0000000..1dd2aaa --- /dev/null +++ b/.gitlab/ci/gitaly.images.yml @@ -0,0 +1,19 @@ +# Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml +.gitaly-matrix: + parallel: + matrix: + - RUBY: ['2.7', '3.0'] + GOLANG: ['1.16', '1.17'] + GIT: ['2.31', '2.33'] + PGBOUNCER: ['1.14', ''] + +gitaly:test: + needs: [] + extends: + - .gitaly-matrix + - .test_custom + +gitaly push: + extends: + - .gitaly-matrix + - .build_and_deploy_custom diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml new file mode 100644 index 0000000..33e7b40 --- /dev/null +++ b/.gitlab/ci/gitlab.images.yml @@ -0,0 +1,46 @@ +# Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml +.gitlab-matrix: + variables: + GOLANG: '1.16' + LFS: '2.9' + CHROME: '89' + NODE: '14.15' + YARN: '1.22' + GRAPHICSMAGICK: '1.3.36' + parallel: + matrix: + - RUBY: ['2.7.patched', '3.0.patched'] + GIT: ['2.31', '2.33'] + POSTGRESQL: ['11', '12', '13'] + +gitlab test: + extends: + - .gitlab-matrix + - .test_custom + +gitlab push: + extends: + - .gitlab-matrix + - .build_and_deploy_custom + +# Used by GitLab's compile-production-assets and compile-test-assets jobs +.gitlab-assets-matrix: + variables: + LFS: '2.9' + NODE: '14.15' + YARN: '1.22' + GRAPHICSMAGICK: '1.3.36' + parallel: + matrix: + - RUBY: ['2.7', '3.0'] + GIT: ['2.31', '2.33'] + +gitlab-assets test: + extends: + - .gitlab-assets-matrix + - .test_custom + +gitlab-assets push: + extends: + - .gitlab-assets-matrix + - .build_and_deploy_custom diff --git a/.gitlab/ci/kas.images.yml b/.gitlab/ci/kas.images.yml new file mode 100644 index 0000000..08cc802 --- /dev/null +++ b/.gitlab/ci/kas.images.yml @@ -0,0 +1,24 @@ +# Used by GitLab/KAS integration tests +.gitlab-kas-matrix: + variables: + GOLANG: '1.16' + LFS: '2.9' + CHROME: '89' + NODE: '14.15' + YARN: '1.22' + GRAPHICSMAGICK: '1.3.36' + GIT: '2.31' + POSTGRESQL: '12' + parallel: + matrix: + - RUBY: ['2.7.2.patched', '2.7.patched', '3.0.patched'] + +gitlab-kas test: + extends: + - .gitlab-kas-matrix + - .test_custom + +gitlab-kas push: + extends: + - .gitlab-kas-matrix + - .build_and_deploy_custom diff --git a/scripts/build-helpers.sh b/scripts/build-helpers.sh deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 9bd8c3d..04445a0 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -1,9 +1,9 @@ #!/bin/bash - set -e IFS=$'\n\t' -source scripts/build-helpers.sh +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +source "$SCRIPT_DIR/lib/custom-docker.sh" function print_golang_args() { case "$1" in @@ -245,23 +245,25 @@ function parse_arguments() { CUSTOM_IMAGE_NAME=debian CUSTOM_IMAGE_VERSION=buster - while read image; do - read version - case "$image" in - ruby) print_ruby_args $version ;; - golang) print_golang_args $version ;; - chrome) print_chrome_args $version ;; - docker) print_docker_args $version ;; - git) print_git_args $version ;; - lfs) print_lfs_args $version ;; - node) print_node_args $version ;; - yarn) print_yarn_args $version ;; - postgresql) print_postgres_args $version ;; - graphicsmagick) print_graphicsmagick_args $version ;; - pgbouncer) print_pgbouncer_args $version ;; - bazelisk) print_bazelisk_args $version ;; - *) exit 1;; - esac + for tool in "${TOOLS[@]}"; do + if [ -n "${!tool}" ]; then + version="${!tool}" + case "$tool" in + RUBY) print_ruby_args $version ;; + GOLANG) print_golang_args $version ;; + CHROME) print_chrome_args $version ;; + DOCKER) print_docker_args $version ;; + GIT) print_git_args $version ;; + LFS) print_lfs_args $version ;; + NODE) print_node_args $version ;; + YARN) print_yarn_args $version ;; + POSTGRESQL) print_postgres_args $version ;; + GRAPHICSMAGICK) print_graphicsmagick_args $version ;; + PGBOUNCER) print_pgbouncer_args $version ;; + BAZELISK) print_bazelisk_args $version ;; + *) echo "unknown tool $tool"; exit 1;; + esac + fi done printf -- "--build-arg CUSTOM_IMAGE_NAME=%s " "$CUSTOM_IMAGE_NAME" @@ -269,10 +271,8 @@ function parse_arguments() { } function generate_command() { - build_image_name=$1; shift; - printf -- "docker build " - echo $build_image_name | tr '-' '\n' | parse_arguments + parse_arguments for i in "$@" do @@ -282,14 +282,17 @@ function generate_command() { } function build_custom_if_needed() { - build_image_name=$1 + build_image_name=$(get_image_name) + echo "Building image $build_image_name" full_image_name="$CI_REGISTRY_IMAGE:$build_image_name" # This re-uses and builds an existing image if needed docker pull --quiet "$full_image" || true docker_command=$(generate_command $@ --cache-from="$full_image_name" ) - echo "Building $build_image_name with $docker_command" + echo "Docker command:" + printf "\t%s" "$docker_command" eval $docker_command + printf "\n\nSUCCESS - Successfully built:\n\t%s" "$build_image_name" } build_custom_if_needed $@ diff --git a/scripts/custom-docker-push b/scripts/custom-docker-push new file mode 100755 index 0000000..37dd870 --- /dev/null +++ b/scripts/custom-docker-push @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +IFS=$'\n\t' + +SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +source "$SCRIPT_DIR/lib/custom-docker.sh" + +function custom_push() { + build_image_name=$(get_image_name) + echo "Pushing image $build_image_name" + full_image_name="$CI_REGISTRY_IMAGE:$build_image_name" + + docker push "$full_image_name" + + printf "\n\nSUCCESS - Successfully pushed:\n\t%s" "$build_image_name" +} + +custom_push diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh new file mode 100644 index 0000000..e8fee8b --- /dev/null +++ b/scripts/lib/custom-docker.sh @@ -0,0 +1,12 @@ +TOOLS=(RUBY GOLANG GIT LFS CHROME NODE YARN POSTGRESQL GRAPHICSMAGICK PGBOUNCER BAZELISK) + +function get_image_name(){ + local IMAGE_NAME + IMAGE_NAME="" + for tool in "${TOOLS[@]}"; do + if [ -n "${!tool}" ]; then + IMAGE_NAME="${IMAGE_NAME}-${tool,,}-${!tool}" + fi + done + echo "${IMAGE_NAME:1}" +}