diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4a5058c..5968ff3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -17,43 +17,57 @@ stages:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_HOST: tcp://docker:2375
-  FORCE_BUILD: "false"
+
+# Run test job always
+.test:
+  stage: test
+  rules:
+    - when: always
+
+# Run build job only on a default branch when triggered manually
+.build:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+      when: manual
 
 .test_build: &test_build
-  stage: test
+  extends: .test
   script:
     # Hack to set an array in /bin/sh
     # http://unix.stackexchange.com/a/137571
     - set -- $CI_JOB_NAME
-    - docker build -f "Dockerfile.$1" .
-  except:
-    - master
+    - docker pull "$CI_REGISTRY_IMAGE:$1" || true
+    - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" .
 
 .build_and_deploy: &build_and_deploy
-  stage: build
-  script:
-    - docker build --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME" -f "Dockerfile.$CI_JOB_NAME" .
-    - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-  only:
-    - master
-
-.test_custom: &test_custom
-  stage: test
+  extends: .build
   script:
     # Hack to set an array in /bin/sh
     # http://unix.stackexchange.com/a/137571
     - set -- $CI_JOB_NAME
-    - ./scripts/custom-docker-build $1
-  except:
-    - master
+    - docker pull "$CI_REGISTRY_IMAGE:$1" || true
+    - docker build --cache-from="$CI_REGISTRY_IMAGE:$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1" -f "Dockerfile.$1" .
+    - docker push "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
+
+.test_custom: &test_custom
+  extends: .test
+  script:
+    # Hack to set an array in /bin/sh
+    # http://unix.stackexchange.com/a/137571
+    - set -- $CI_JOB_NAME
+    - ./scripts/custom-docker-build "$1"
+  rules:
+    - when: always
 
 .build_and_deploy_custom: &build_and_deploy_custom
-  stage: build
+  extends: .build
   script:
-    - ./scripts/custom-docker-build $CI_JOB_NAME --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-    - push_if_needed "$CI_REGISTRY_IMAGE:$CI_JOB_NAME"
-  only:
-    - master
+    # Hack to set an array in /bin/sh
+    # http://unix.stackexchange.com/a/137571
+    - set -- $CI_JOB_NAME
+    - ./scripts/custom-docker-build "$1" --label "ci_pipeline_url=$CI_PIPELINE_URL" --label "ci_job_url=$CI_JOB_URL" -t "$CI_REGISTRY_IMAGE:$1"
+    - docker push "$CI_REGISTRY_IMAGE:$1"
 
 # Tests
 
diff --git a/Dockerfile.custom b/Dockerfile.custom
index 2b6c8bf..0693d6b 100644
--- a/Dockerfile.custom
+++ b/Dockerfile.custom
@@ -30,16 +30,19 @@ RUN if [ -n "$GIT_VERSION" ]; then /scripts/install-git && git --version; fi
 # Chrome
 ARG CHROME_VERSION
 ARG CHROME_DRIVER_VERSION
+
 RUN if [ -n "$CHROME_VERSION" ]; then /scripts/install-chrome $CHROME_VERSION $CHROME_DRIVER_VERSION && google-chrome --version; fi
 
 # NodeJS and Yarn
 ARG NODE_INSTALL_VERSION
 ARG YARN_INSTALL_VERSION
+
 RUN if [ -n "$NODE_INSTALL_VERSION" ] ; then /scripts/install-node $NODE_INSTALL_VERSION $YARN_INSTALL_VERSION && node --version && yarn --version; fi
 
 # Golang
 ARG INSTALL_GOLANG_VERSION
 ARG GOLANG_DOWNLOAD_SHA256
+
 RUN if [ -n "$INSTALL_GOLANG_VERSION" ] ; then /scripts/install-golang "${INSTALL_GOLANG_VERSION}" "${GOLANG_DOWNLOAD_SHA256}" && go version; fi
 
 # Git LFS (https://git-lfs.github.com/)
@@ -51,33 +54,54 @@ RUN if [ -n "$LFS_VERSION" ]; then /scripts/install-lfs && git lfs --version; fi
 
 # Postgres
 ARG POSTGRES_VERSION
+
 RUN if [ -n "$POSTGRES_VERSION" ] ; then /scripts/install-postgresql $POSTGRES_VERSION; fi
 
 # Ansible
 ARG ANSIBLE_VERSION
+
 RUN if [ -n "$ANSIBLE_VERSION" ] ; then /scripts/install-ansible $ANSIBLE_VERSION; fi
 
 # Terraform
 ARG TERRAFORM_VERSION
 ARG TERRAFORM_DOWNLOAD_SHA256
+
 RUN if [ -n "$TERRAFORM_VERSION" ] ; then /scripts/install-terraform $TERRAFORM_VERSION $TERRAFORM_DOWNLOAD_SHA256; fi
 
 # GraphicsMagick
 ARG GRAPHISMAGICK_VERSION
 ARG GRAPHISMAGICK_DOWNLOAD_URL=https://sourceforge.net/projects/graphicsmagick/files/graphicsmagick/${GRAPHISMAGICK_VERSION}/GraphicsMagick-${GRAPHISMAGICK_VERSION}.tar.gz
 ARG GRAPHISMAGICK_DOWNLOAD_SHA256
+
 RUN if [ -n "$GRAPHISMAGICK_VERSION" ]; then /scripts/install-graphicsmagick && gm version; fi
 
 # Docker
 ARG DOCKER_VERSION
+
 RUN if [ -n "$DOCKER_VERSION" ]; then /scripts/install-docker $DOCKER_VERSION; fi
 
 # PgBouncer
 ARG PGBOUNCER_VERSION
 ARG PGBOUNCER_DOWNLOAD_SHA256
+
 RUN if [ -n "$PGBOUNCER_VERSION" ] ; then /scripts/install-pgbouncer $PGBOUNCER_VERSION $PGBOUNCER_DOWNLOAD_SHA256; fi
 
 RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+ENV LANG=en_US.UTF-8 \
+  LANGUAGE=en_US:en \
+  LC_ALL=en_US.UTF-8
+
+# Set as env variables all versions configured
+ENV RUBY_VERSION=${RUBY_VERSION} \
+  GIT_VERSION=${GIT_VERSION} \
+  CHROME_VERSION=${CHROME_VERSION} \
+  NODE_VERSION=${NODE_INSTALL_VERSION} \
+  YARN_VERSION=${YARN_INSTALL_VERSION} \
+  GOLANG_VERSION=${INSTALL_GOLANG_VERSION} \
+  LFS_VERSION=${LFS_VERSION} \
+  POSTGRES_VERSION=${POSTGRES_VERSION} \
+  ANSIBLE_VERSION=${ANSIBLE_VERSION} \
+  TERRAFORM_VERSION=${TERRAFORM_VERSION} \
+  GRAPHISMAGICK_VERSION=${GRAPHISMAGICK_VERSION} \
+  DOCKER_VERSION=${DOCKER_VERSION} \
+  PGBOUNCER_VERSION=${PGBOUNCER_VERSION}
diff --git a/README.md b/README.md
index 71dc6a7..24b4137 100644
--- a/README.md
+++ b/README.md
@@ -66,17 +66,13 @@ the name would be `ruby-2.4-golang-1.9-git-2.14`.
 1. Add a test task: `ruby-2.4-golang-1.9-git-2.14 test: *test_custom`
 1. Add a new build task: `ruby-2.4-golang-1.9-git-2.14: *build_and_deploy_custom`
 
-#### Forcing custom images to be rebuilt
+#### Pushing a rebuild image
 
-By default, once a custom image is built, tagged and pushed to the registry, it's
-not rebuilt to ensure an upstream dependency doesn't end up breaking our images
-unexpectedly.
+To build (or rebuild) a given image and push it, you need to enable manual action for a given
+job after it is merged to master.
 
-For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192
-
-In the rare case where the `Dockerfile.custom` file is updated and all custom
-images shoulld be rebuild, you can start a new pipeline and set the variable
-`FORCE_BUILD` to `true`.
+By default we don't do it to ensure that an upstream dependency doesn't end up breaking our images
+unexpectedly. For reference, this happened in the past: https://gitlab.com/gitlab-org/gitlab/issues/205192
 
 ## Note regarding Google Chrome
 
diff --git a/scripts/build-helpers.sh b/scripts/build-helpers.sh
index 52b25b0..e69de29 100644
--- a/scripts/build-helpers.sh
+++ b/scripts/build-helpers.sh
@@ -1,25 +0,0 @@
-function image_already_exists() {
-    image=$1
-
-    $(docker pull $image > /dev/null)
-}
-
-function build_if_needed() {
-    image=$1
-
-    if ! image_already_exists $image; then
-        docker build -t "$image" -f "Dockerfile.$image" .
-    else
-        echo "$image already exists, skipping build."
-    fi
-}
-
-function push_if_needed() {
-    image=$1
-
-    if ! image_already_exists $image; then
-        docker push "$image"
-    else
-        echo "$image already exists, skipping push."
-    fi
-}
diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build
index a3b60ff..d2e8fd2 100755
--- a/scripts/custom-docker-build
+++ b/scripts/custom-docker-build
@@ -303,17 +303,11 @@ function build_custom_if_needed() {
     build_image_name=$1
     full_image_name="$CI_REGISTRY_IMAGE:$build_image_name"
 
-    if [[ "$FORCE_BUILD" == "true" ]] || ! image_already_exists $full_image_name; then
-      docker_command=$(generate_command $@)
-      if [[ "$FORCE_BUILD" == "true" ]]; then
-        echo "Force building $build_image_name due to \$FORCE_BUILD=true with $docker_command"
-      else
-        echo "Building $build_image_name with $docker_command"
-      fi
-      eval $docker_command
-    else
-      echo "$build_image_name already exists, skipping build."
-    fi
+    # This re-uses and builds an existing image if needed
+    docker pull "$full_image" || true
+    docker_command=$(generate_command $@ --cache-from="$full_image_name" )
+    echo "Building $build_image_name with $docker_command"
+    eval $docker_command
 }
 
 build_custom_if_needed $@