Merge branch '36-build-only-the-images-whose-dockerfiles-are-changed' into 'master'

ci: Only build custom images that need to be rebuilt Closes #36 See merge request https://gitlab.com/gitlab-org/gitlab-build-images/-/merge_requests/633 Merged-by: Stan Hu <stanhu@gmail.com> Approved-by: Stan Hu <stanhu@gmail.com> Reviewed-by: Rémy Coutable <remy@rymai.me> Co-authored-by: Rémy Coutable <remy@rymai.me>
2025-12-09 18:12:55 +01:00 · 2023-01-25 20:44:07 +00:00 · 2023-01-25 20:44:07 +00:00 · 3368a86d26
commit 3368a86d26
parent ccd8e00291 684f0ce8ef
10 changed files with 125 additions and 62 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -5,7 +5,13 @@ include:
 workflow:
  rules:
    - if: $CI_MERGE_REQUEST_IID
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"'
+      variables:
+        PUSH_CUSTOM_IMAGE: "true"
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_SERVER_HOST == "ops.gitlab.net" || $CI_SERVER_HOST == "dev.gitlab.org")'
+      variables:
+        PUSH_CUSTOM_IMAGE: "true"
+        BASE_BUILD_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE

 default:
  interruptible: true
--- a/.gitlab/ci/custom.images.yml
+++ b/.gitlab/ci/custom.images.yml
@ -1,72 +1,54 @@
-.build_and_deploy:
-  extends: .build_and_push
-  script:
-    # Hack to set an array in /bin/sh
-    # http://unix.stackexchange.com/a/137571
-    - set -- $CI_JOB_NAME
-    - |
-      docker buildx build \
-        --cache-to=type=inline \
-        --cache-from="$CI_REGISTRY_IMAGE:$1" \
-        --platform=linux/${ARCH:-amd64} \
-        --push="${PUSH_CUSTOM_IMAGE:-false}" \
-        --label "ci_pipeline_url=$CI_PIPELINE_URL" \
-        --label "ci_job_url=$CI_JOB_URL" \
-        -t "$CI_REGISTRY_IMAGE:$1" \
-        -f "Dockerfile.$1" \
-        .
-
 danger:
-  extends: .build_and_deploy
+  extends: .build_static_image

 gitlab-charts-build-base-helm-3.7:
-  extends: .build_and_deploy
+  extends: .build_static_image

 gitlab-helm3.5-kubectl1.17:
-  extends: .build_and_deploy
+  extends: .build_static_image

 gitlab-vscode-nodeless:
-  extends: .build_and_deploy
+  extends: .build_static_image

 kaniko:
-  extends: .build_and_deploy
+  extends: .build_static_image

 release-tools:
-  extends: .build_and_deploy
+  extends: .build_static_image

 triage-ops-ruby-2.7:
-  extends: .build_and_deploy
+  extends: .build_static_image

 sitespeed-gitlab:
-  extends: .build_and_deploy
+  extends: .build_static_image

 ubi-release:
-  extends: .build_and_deploy
+  extends: .build_static_image

 www-gitlab-com-debian-bullseye-ruby-3.0-node-16:
-  extends: .build_and_deploy
+  extends: .build_static_image

 build-git:
-  extends: .build_and_deploy
+  extends: .build_static_image

 # Used by GitLab: https://gitlab.com/gitlab-org/gitlab/-/blob/13-8-stable-ee/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
 terraform:
-  extends: .build_and_deploy
+  extends: .build_static_image

 # Used by go projects for linting https://gitlab.com/gitlab-org/gitlab/blob/master/doc/development/go_guide/index.md#automatic-linting
 golangci-lint-alpine:
-  extends: .build_and_deploy
+  extends: .build_static_image

 # Used by gitlab-operator: https://gitlab.com/gitlab-org/cloud-native/gitlab-operator
 gitlab-operator-build-base-golang-1.18:
-  extends: .build_and_deploy
+  extends: .build_static_image

 alpine-bash-jq-curl-git:
-  extends: .build_and_deploy
+  extends: .build_static_image

 # Used by frontend related tasks for linting GraphQL
 apollo:
-  extends: .build_and_deploy
+  extends: .build_static_image

 zoekt-ci-image:
-  extends: .build_and_deploy
+  extends: .build_static_image
--- a/.gitlab/ci/customers.images.yml
+++ b/.gitlab/ci/customers.images.yml
@ -1,7 +1,16 @@
+.customers:
+  extends:
+    - .build_dynamic_image
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/customers.images.yml"
+
 # Used by customers-gitlab-com rspec job: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab-ci.yml
 customers:
  extends:
-    - .build_and_push
+    - .customers
  variables:
    LFS: '2.9'
    CHROME: '107'
@ -17,7 +26,7 @@ customers:
 # Used by customers-gitlab-com qa e2e test: https://gitlab.com/gitlab-org/customers-gitlab-com/-/blob/staging/.gitlab/ci/e2e.gitlab-ci.yml
 customers-qa-assets:
  extends:
-    - .build_and_push
+    - .customers
  variables:
    CHROME: '107'
    NODE: '16.14'
--- a/.gitlab/ci/definitions.yml
+++ b/.gitlab/ci/definitions.yml
@ -19,7 +19,7 @@
      fi
    - docker buildx create --use # creates and set's to active buildkit builder

-.build_and_push:
+.build_dynamic_image:
  stage: custom
  extends: .install-qemu
  needs: []
@ -28,14 +28,38 @@
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_SERVER_HOST == "gitlab.com"'
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"'
-      variables:
-        PUSH_CUSTOM_IMAGE: "true"
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && ($CI_SERVER_HOST == "ops.gitlab.net" || $CI_SERVER_HOST == "dev.gitlab.org")'
-      variables:
-        PUSH_CUSTOM_IMAGE: "true"
-        BASE_BUILD_REGISTRY_IMAGE: $CI_REGISTRY_IMAGE
+
+.build_static_image:
+  stage: custom
+  extends: .install-qemu
+  needs: []
+  script:
+    # Hack to set an array in /bin/sh
+    # http://unix.stackexchange.com/a/137571
+    - set -- $CI_JOB_NAME
+    - |
+      docker buildx build \
+        --cache-to=type=inline \
+        --cache-from="$CI_REGISTRY_IMAGE:$1" \
+        --platform=linux/${ARCH:-amd64} \
+        --push="${PUSH_CUSTOM_IMAGE:-false}" \
+        --label "ci_pipeline_url=$CI_PIPELINE_URL" \
+        --label "ci_job_url=$CI_JOB_URL" \
+        -t "$CI_REGISTRY_IMAGE:$1" \
+        -f "Dockerfile.$1" \
+        .
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_MERGE_REQUEST_IID'
+      changes:
+        - "Dockerfile.${CI_JOB_NAME}"
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      changes:
+        - "Dockerfile.${CI_JOB_NAME}"
+    # In all other cases, make the job manual
+    - when: manual
+      allow_failure: true

 .cache-google-chrome:
  stage: automation
@ -63,9 +87,11 @@
        -f "Dockerfile.cache-chrome" \
        .
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_SERVER_HOST == "gitlab.com"'
-    - if: '$CI_PIPELINE_SOURCE == "schedule" && $CI_SERVER_HOST == "gitlab.com"'
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_SERVER_HOST == "gitlab.com"'
+    - if: '$CI_SERVER_HOST != "gitlab.com"'
+      when: never
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+    - if: '$CI_MERGE_REQUEST_IID'
      when: manual
      allow_failure: true

--- a/.gitlab/ci/docker.images.yml
+++ b/.gitlab/ci/docker.images.yml
@ -1,19 +1,24 @@
 # Docker image with buildx plugin
 .docker:
+  extends:
+    - .build_dynamic_image
  variables:
    DOCKER: '20.10.21'
    BUILDX: '0.9'
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/docker.images.yml"

 docker:
  extends:
    - .docker
-    - .build_and_push
  variables:
    OS: "debian:bullseye"

 docker-slim:
  extends:
    - .docker
-    - .build_and_push
  variables:
    OS: "debian:bullseye-slim"
--- a/.gitlab/ci/e2e.images.yml
+++ b/.gitlab/ci/e2e.images.yml
@ -1,9 +1,17 @@
 # Used by gitlab e2e tests and other qa related tasks
+.e2e:
+  extends:
+    - .build_dynamic_image
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/e2e.images.yml"

 # Base image with just ruby and bundler
 e2e:
  extends:
-    - .build_and_push
+    - .e2e
  variables:
    OS: "debian:bullseye"
    BUNDLER: '2.3'
--- a/.gitlab/ci/gitaly.images.yml
+++ b/.gitlab/ci/gitaly.images.yml
@ -1,8 +1,17 @@
 # Used by Gitaly: https://gitlab.com/gitlab-org/gitaly/blob/master/.gitlab-ci.yml
+.gitaly:
+  extends:
+    - .build_dynamic_image
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/gitaly.images.yml"
+
 gitaly:
  needs: []
  extends:
-    - .build_and_push
+    - .gitaly
  stage: gitaly
  parallel:
    matrix:
--- a/.gitlab/ci/gitlab.images.yml
+++ b/.gitlab/ci/gitlab.images.yml
@ -1,7 +1,17 @@
 # Used by GitLab: https://gitlab.com/gitlab-org/gitlab/blob/master/.gitlab-ci.yml
+.gitlab:
+  extends:
+    - .build_dynamic_image
+  stage: gitlab
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/gitlab.images.yml"
+
 gitlab:
  extends:
-    - .build_and_push
+    - .gitlab
  stage: gitlab
  variables:
    RUBYGEMS: '3.2'
@ -22,8 +32,7 @@ gitlab:

 gitlab-ruby-3.2:
  extends:
-    - .build_and_push
-  stage: gitlab
+    - .gitlab
  variables:
    RUBYGEMS: '3.4'
    LFS: '2.9'
@ -44,7 +53,7 @@ gitlab-ruby-3.2:
 # Used by GitLab's compile-production-assets and compile-test-assets jobs
 gitlab-assets:
  extends:
-    - .build_and_push
+    - .gitlab
  stage: gitlab-assets
  variables:
    RUBYGEMS: '3.2'
@ -62,7 +71,7 @@ gitlab-assets:
 # Used by GitLab's Review app jobs
 gitlab-review-app:
  extends:
-    - .build_and_push
+    - .gitlab
  stage: gitlab-review-app
  variables:
    RUBY: '3.0'
--- a/.gitlab/ci/kas.images.yml
+++ b/.gitlab/ci/kas.images.yml
@ -1,7 +1,16 @@
 # Used by GitLab/KAS integration tests
+.kas:
+  extends:
+    - .build_dynamic_image
+  rules:
+    - !reference [.build_dynamic_image, rules]
+    - changes:
+        - "scripts/lib/custom-docker-build"
+        - ".gitlab/ci/kas.images.yml"
+
 gitlab-kas:
  extends:
-    - .build_and_push
+    - .kas
  variables:
    GOLANG: '1.19'
    RUST: '1.65.0'
--- a/README.md
+++ b/README.md
@ -27,7 +27,7 @@ Assuming the image you want to add is called `new-image`:

 ```yml
 new-image:
-  extends: .build_and_deploy
+  extends: .build_static_image
 ```

 1. Add a Dockerfile: `Dockerfile.new-image`.
@ -84,7 +84,7 @@ following job definition would have to be created:
 ```yml
 new_image:
  extends:
-    - .build_and_push
+    - .build_dynamic_image
  variables:
    OS: 'debian:bullseye'
    RUBY: '3.0'