From 684f0ce8efc586773ce20cdd15fb0d2ee08e8eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 25 Jan 2023 18:39:28 +0100 Subject: [PATCH] ci: Only build custom images that need to be rebuilt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 8 ++++- .gitlab/ci/custom.images.yml | 52 +++++++++++---------------------- .gitlab/ci/customers.images.yml | 13 +++++++-- .gitlab/ci/definitions.yml | 50 +++++++++++++++++++++++-------- .gitlab/ci/docker.images.yml | 9 ++++-- .gitlab/ci/e2e.images.yml | 10 ++++++- .gitlab/ci/gitaly.images.yml | 11 ++++++- .gitlab/ci/gitlab.images.yml | 19 ++++++++---- .gitlab/ci/kas.images.yml | 11 ++++++- README.md | 4 +-- 10 files changed, 125 insertions(+), 62 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3d2f441..3f67395 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,13 @@ include: workflow: rules: - if: $CI_MERGE_REQUEST_IID - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"' + variables: + PUSH_CUSTOM_IMAGE: "true" + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_SERVER_HOST == "ops.gitlab.net" || $CI_SERVER_HOST == "dev.gitlab.org")' + variables: + PUSH_CUSTOM_IMAGE: "true" + BASE_BUILD_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE default: interruptible: true diff --git a/.gitlab/ci/custom.images.yml b/.gitlab/ci/custom.images.yml index 0eaf913..24b03f6 100644 --- a/.gitlab/ci/custom.images.yml +++ b/.gitlab/ci/custom.images.yml @@ -1,72 +1,54 @@ -.build_and_deploy: - extends: .build_and_push - script: - # Hack to set an array in /bin/sh - # http://unix.stackexchange.com/a/137571 - - set -- $CI_JOB_NAME - - | - docker buildx build \ - --cache-to=type=inline \ - --cache-from="$CI_REGISTRY_IMAGE:$1" \ - --platform=linux/${ARCH:-amd64} \ - --push="${PUSH_CUSTOM_IMAGE:-false}" \ - --label "ci_pipeline_url=$CI_PIPELINE_URL" \ - --label "ci_job_url=$CI_JOB_URL" \ - -t "$CI_REGISTRY_IMAGE:$1" \ - -f "Dockerfile.$1" \ - . - danger: - extends: .build_and_deploy + extends: .build_static_image gitlab-charts-build-base-helm-3.7: - extends: .build_and_deploy + extends: .build_static_image gitlab-helm3.5-kubectl1.17: - extends: .build_and_deploy + extends: .build_static_image gitlab-vscode-nodeless: - extends: .build_and_deploy + extends: .build_static_image kaniko: - extends: .build_and_deploy + extends: .build_static_image release-tools: - extends: .build_and_deploy + extends: .build_static_image triage-ops-ruby-2.7: - extends: .build_and_deploy + extends: .build_static_image sitespeed-gitlab: - extends: .build_and_deploy + extends: .build_static_image ubi-release: - extends: .build_and_deploy + extends: .build_static_image www-gitlab-com-debian-bullseye-ruby-3.0-node-16: - extends: .build_and_deploy + extends: .build_static_image build-git: - extends: .build_and_deploy + extends: .build_static_image # Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml terraform: - extends: .build_and_deploy + extends: .build_static_image # Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting golangci-lint-alpine: - extends: .build_and_deploy + extends: .build_static_image # Used by gitlab-operator: https://gitlab.com/gitlab-org/cloud-native/gitlab-operator gitlab-operator-build-base-golang-1.18: - extends: .build_and_deploy + extends: .build_static_image alpine-bash-jq-curl-git: - extends: .build_and_deploy + extends: .build_static_image # Used by frontend related tasks for linting GraphQL apollo: - extends: .build_and_deploy + extends: .build_static_image zoekt-ci-image: - extends: .build_and_deploy + extends: .build_static_image diff --git a/.gitlab/ci/customers.images.yml b/.gitlab/ci/customers.images.yml index e4520b5..000625e 100644 --- a/.gitlab/ci/customers.images.yml +++ b/.gitlab/ci/customers.images.yml @@ -1,7 +1,16 @@ +.customers: + extends: + - .build_dynamic_image + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/customers.images.yml" + # Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml customers: extends: - - .build_and_push + - .customers variables: LFS: '2.9' CHROME: '107' @@ -17,7 +26,7 @@ customers: # Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml customers-qa-assets: extends: - - .build_and_push + - .customers variables: CHROME: '107' NODE: '16.14' diff --git a/.gitlab/ci/definitions.yml b/.gitlab/ci/definitions.yml index cd1cdb4..18c0322 100644 --- a/.gitlab/ci/definitions.yml +++ b/.gitlab/ci/definitions.yml @@ -19,7 +19,7 @@ fi - docker buildx create --use # creates and set's to active buildkit builder -.build_and_push: +.build_dynamic_image: stage: custom extends: .install-qemu needs: [] @@ -28,14 +28,38 @@ rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_SERVER_HOST == "gitlab.com"' - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"' - variables: - PUSH_CUSTOM_IMAGE: "true" - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_SERVER_HOST == "ops.gitlab.net" || $CI_SERVER_HOST == "dev.gitlab.org")' - variables: - PUSH_CUSTOM_IMAGE: "true" - BASE_BUILD_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE + +.build_static_image: + stage: custom + extends: .install-qemu + needs: [] + script: + # Hack to set an array in /bin/sh + # http://unix.stackexchange.com/a/137571 + - set -- $CI_JOB_NAME + - | + docker buildx build \ + --cache-to=type=inline \ + --cache-from="$CI_REGISTRY_IMAGE:$1" \ + --platform=linux/${ARCH:-amd64} \ + --push="${PUSH_CUSTOM_IMAGE:-false}" \ + --label "ci_pipeline_url=$CI_PIPELINE_URL" \ + --label "ci_job_url=$CI_JOB_URL" \ + -t "$CI_REGISTRY_IMAGE:$1" \ + -f "Dockerfile.$1" \ + . + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_MERGE_REQUEST_IID' + changes: + - "Dockerfile.${CI_JOB_NAME}" + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + changes: + - "Dockerfile.${CI_JOB_NAME}" + # In all other cases, make the job manual + - when: manual + allow_failure: true .cache-google-chrome: stage: automation @@ -63,9 +87,11 @@ -f "Dockerfile.cache-chrome" \ . rules: - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"' - - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_SERVER_HOST == "gitlab.com"' - - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_SERVER_HOST == "gitlab.com"' + - if: '$CI_SERVER_HOST != "gitlab.com"' + when: never + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_PIPELINE_SOURCE == "schedule"' + - if: '$CI_MERGE_REQUEST_IID' when: manual allow_failure: true diff --git a/.gitlab/ci/docker.images.yml b/.gitlab/ci/docker.images.yml index d01e36c..1f49b90 100644 --- a/.gitlab/ci/docker.images.yml +++ b/.gitlab/ci/docker.images.yml @@ -1,19 +1,24 @@ # Docker image with buildx plugin .docker: + extends: + - .build_dynamic_image variables: DOCKER: '20.10.21' BUILDX: '0.9' + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/docker.images.yml" docker: extends: - .docker - - .build_and_push variables: OS: "debian:bullseye" docker-slim: extends: - .docker - - .build_and_push variables: OS: "debian:bullseye-slim" diff --git a/.gitlab/ci/e2e.images.yml b/.gitlab/ci/e2e.images.yml index fe3f4e7..da7915f 100644 --- a/.gitlab/ci/e2e.images.yml +++ b/.gitlab/ci/e2e.images.yml @@ -1,9 +1,17 @@ # Used by gitlab e2e tests and other qa related tasks +.e2e: + extends: + - .build_dynamic_image + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/e2e.images.yml" # Base image with just ruby and bundler e2e: extends: - - .build_and_push + - .e2e variables: OS: "debian:bullseye" BUNDLER: '2.3' diff --git a/.gitlab/ci/gitaly.images.yml b/.gitlab/ci/gitaly.images.yml index 9335e70..bf41501 100644 --- a/.gitlab/ci/gitaly.images.yml +++ b/.gitlab/ci/gitaly.images.yml @@ -1,8 +1,17 @@ # Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml +.gitaly: + extends: + - .build_dynamic_image + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/gitaly.images.yml" + gitaly: needs: [] extends: - - .build_and_push + - .gitaly stage: gitaly parallel: matrix: diff --git a/.gitlab/ci/gitlab.images.yml b/.gitlab/ci/gitlab.images.yml index 14b8a66..ffd6be2 100644 --- a/.gitlab/ci/gitlab.images.yml +++ b/.gitlab/ci/gitlab.images.yml @@ -1,7 +1,17 @@ # Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml +.gitlab: + extends: + - .build_dynamic_image + stage: gitlab + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/gitlab.images.yml" + gitlab: extends: - - .build_and_push + - .gitlab stage: gitlab variables: RUBYGEMS: '3.2' @@ -22,8 +32,7 @@ gitlab: gitlab-ruby-3.2: extends: - - .build_and_push - stage: gitlab + - .gitlab variables: RUBYGEMS: '3.4' LFS: '2.9' @@ -44,7 +53,7 @@ gitlab-ruby-3.2: # Used by GitLab's compile-production-assets and compile-test-assets jobs gitlab-assets: extends: - - .build_and_push + - .gitlab stage: gitlab-assets variables: RUBYGEMS: '3.2' @@ -62,7 +71,7 @@ gitlab-assets: # Used by GitLab's Review app jobs gitlab-review-app: extends: - - .build_and_push + - .gitlab stage: gitlab-review-app variables: RUBY: '3.0' diff --git a/.gitlab/ci/kas.images.yml b/.gitlab/ci/kas.images.yml index 677dc4b..e5b0639 100644 --- a/.gitlab/ci/kas.images.yml +++ b/.gitlab/ci/kas.images.yml @@ -1,7 +1,16 @@ # Used by GitLab/KAS integration tests +.kas: + extends: + - .build_dynamic_image + rules: + - !reference [.build_dynamic_image, rules] + - changes: + - "scripts/lib/custom-docker-build" + - ".gitlab/ci/kas.images.yml" + gitlab-kas: extends: - - .build_and_push + - .kas variables: GOLANG: '1.19' RUST: '1.65.0' diff --git a/README.md b/README.md index deb3f69..fb74dbf 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Assuming the image you want to add is called `new-image`: ```yml new-image: - extends: .build_and_deploy + extends: .build_static_image ``` 1. Add a Dockerfile: `Dockerfile.new-image`. @@ -84,7 +84,7 @@ following job definition would have to be created: ```yml new_image: extends: - - .build_and_push + - .build_dynamic_image variables: OS: 'debian:bullseye' RUBY: '3.0'