diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4a5058c..6c44a60 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,43 +17,51 @@ stages:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_HOST: tcp://docker:2375
-  FORCE_BUILD: "false"
+
+# Run test job always
+.test:
+  stage: test
+  rules:
+    - when: always
+
+# Run build job only on a default branch when triggered manually
+.build:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: manual  
 
 .test_build: &test_build
-  stage: test
+  extends: .test
   script:
     # Hack to set an array in /bin/sh
     # http://unix.stackexchange.com/a/137571
     - set -- $CI_JOB_NAME
-    - docker build -f "Dockerfile.$1" .
-  except:
-    - master
+    - docker pull "$CI_REGISTRY_IMAGE:$1" || true
+    - docker build --from-cache="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" .
 
 .build_and_deploy: &build_and_deploy
-  stage: build
+  extends: .build
   script:
-    - docker build --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" -f "Dockerfile.$CI_JOB_NAME" .
+    - docker pull "$CI_REGISTRY_IMAGE:$1" || true
+    - docker build --from-cache="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" -f "Dockerfile.$CI_JOB_NAME" .
     - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-  only:
-    - master
 
 .test_custom: &test_custom
-  stage: test
+  extends: .test
   script:
     # Hack to set an array in /bin/sh
     # http://unix.stackexchange.com/a/137571
     - set -- $CI_JOB_NAME
     - ./scripts/custom-docker-build $1
-  except:
-    - master
+  rules:
+    - when: always
 
 .build_and_deploy_custom: &build_and_deploy_custom
-  stage: build
+  extends: .build
   script:
     - ./scripts/custom-docker-build $CI_JOB_NAME --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-    - push_if_needed "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-  only:
-    - master
+    - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
 
 # Tests
 
diff --git a/README.md b/README.md
index 71dc6a7..24b4137 100644
--- a/README.md
+++ b/README.md
@@ -66,17 +66,13 @@ the name would be `ruby-2.4-golang-1.9-git-2.14`.
 1. Add a test task: `ruby-2.4-golang-1.9-git-2.14 test: *test_custom`
 1. Add a new build task: `ruby-2.4-golang-1.9-git-2.14: *build_and_deploy_custom`
 
-#### Forcing custom images to be rebuilt
+#### Pushing a rebuild image
 
-By default, once a custom image is built, tagged and pushed to the registry, it's
-not rebuilt to ensure an upstream dependency doesn't end up breaking our images
-unexpectedly.
+To build (or rebuild) a given image and push it, you need to enable manual action for a given
+job after it is merged to master.
 
-For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192
-
-In the rare case where the `Dockerfile.custom` file is updated and all custom
-images shoulld be rebuild, you can start a new pipeline and set the variable
-`FORCE_BUILD` to `true`.
+By default we don't do it to ensure that an upstream dependency doesn't end up breaking our images
+unexpectedly. For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192
 
 ## Note regarding Google Chrome
 
diff --git a/scripts/build-helpers.sh b/scripts/build-helpers.sh
index 52b25b0..e69de29 100644
--- a/scripts/build-helpers.sh
+++ b/scripts/build-helpers.sh
@@ -1,25 +0,0 @@
-function image_already_exists() {
-    image=$1
-
-    $(docker pull $image > /dev/null)
-}
-
-function build_if_needed() {
-    image=$1
-
-    if ! image_already_exists $image; then
-        docker build -t "$image" -f "Dockerfile.$image" .
-    else
-        echo "$image already exists, skipping build."
-    fi
-}
-
-function push_if_needed() {
-    image=$1
-
-    if ! image_already_exists $image; then
-        docker push "$image"
-    else
-        echo "$image already exists, skipping push."
-    fi
-}
diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build
index a3b60ff..9b2ea61 100755
--- a/scripts/custom-docker-build
+++ b/scripts/custom-docker-build
@@ -303,17 +303,11 @@ function build_custom_if_needed() {
     build_image_name=$1
     full_image_name="$CI_REGISTRY_IMAGE:$build_image_name"
 
-    if [[ "$FORCE_BUILD" == "true" ]] || ! image_already_exists $full_image_name; then
-      docker_command=$(generate_command $@)
-      if [[ "$FORCE_BUILD" == "true" ]]; then
-        echo "Force building $build_image_name due to \$FORCE_BUILD=true with $docker_command"
-      else
-        echo "Building $build_image_name with $docker_command"
-      fi
-      eval $docker_command
-    else
-      echo "$build_image_name already exists, skipping build."
-    fi
+    # This re-uses and builds an existing image if needed
+    docker pull "$full_image" || true
+    docker_command=$(generate_command --from-cache="$full_image_name" $@)
+    echo "Building $build_image_name with $docker_command"
+    eval $docker_command
 }
 
 build_custom_if_needed $@