diff --git a/.gitlab/ci/docker.images.yml b/.gitlab/ci/docker.images.yml
index 1f49b90..122c988 100644
--- a/.gitlab/ci/docker.images.yml
+++ b/.gitlab/ci/docker.images.yml
@@ -3,8 +3,7 @@
   extends:
     - .build_dynamic_image
   variables:
-    DOCKER: '20.10.21'
-    BUILDX: '0.9'
+    DOCKER: '23.0.1'
   rules:
     - !reference [.build_dynamic_image, rules]
     - changes:
@@ -14,11 +13,6 @@
 docker:
   extends:
     - .docker
-  variables:
-    OS: "debian:bullseye"
-
-docker-slim:
-  extends:
-    - .docker
-  variables:
-    OS: "debian:bullseye-slim"
+  parallel:
+    matrix:
+      - OS: ['debian:bullseye', 'debian:bullseye-slim']
diff --git a/.gitlab/ci/e2e.images.yml b/.gitlab/ci/e2e.images.yml
index 761c2ee..fae8f2a 100644
--- a/.gitlab/ci/e2e.images.yml
+++ b/.gitlab/ci/e2e.images.yml
@@ -24,7 +24,7 @@ e2e-docker:
   extends:
     - e2e
   variables:
-    DOCKER: '20.10.14'
+    DOCKER: '23.0.1'
 
 # Image with additional chrome installed
 e2e-chrome:
diff --git a/Dockerfile.custom b/Dockerfile.custom
index 8be9021..2bcb798 100644
--- a/Dockerfile.custom
+++ b/Dockerfile.custom
@@ -81,12 +81,6 @@ ARG DOCKER_VERSION
 
 RUN if [ -n "$DOCKER_VERSION" ]; then /scripts/install-docker "$DOCKER_VERSION"; fi
 
-# Docker buildx plugin
-ARG BUILDX_VERSION
-ARG BUILDX_DOWNLOAD_SHA256
-
-RUN if [ -n "$BUILDX_VERSION" ]; then /scripts/install-buildx "$BUILDX_VERSION" "$BUILDX_DOWNLOAD_SHA256"; fi
-
 # Bazelisk
 ARG BAZELISK_VERSION
 ARG BAZELISK_DOWNLOAD_SHA256
diff --git a/scripts/install-buildx b/scripts/install-buildx
deleted file mode 100755
index a2be0ec..0000000
--- a/scripts/install-buildx
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -xeou pipefail
-
-BUILDX_VERSION=${1}
-BUILDX_DOWNLOAD_SHA256=${2}
-BUILDX_DOWNLOAD_URL="https://github.com/docker/buildx/releases/download/v${BUILDX_VERSION}/buildx-v${BUILDX_VERSION}.linux-${TARGETARCH:-amd64}"
-
-/scripts/download-file docker-buildx "$BUILDX_DOWNLOAD_URL" $BUILDX_DOWNLOAD_SHA256
-
-mkdir -p /usr/local/lib/docker/cli-plugins
-chmod +x docker-buildx && mv docker-buildx /usr/local/lib/docker/cli-plugins/
diff --git a/scripts/install-docker b/scripts/install-docker
index 0902fa8..2b0f021 100755
--- a/scripts/install-docker
+++ b/scripts/install-docker
@@ -7,22 +7,26 @@ export DEBIAN_FRONTEND=noninteractive
 DOCKER_VERSION=${1}
 DEBIAN_VERSION=$(lsb_release -c -s)
 
-apt-get update
-apt-get -y install \
+apt-get update && apt-get -y install \
     apt-transport-https \
     ca-certificates \
     curl \
     gnupg
 
-curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -
-
-
-echo "deb https://download.docker.com/linux/debian ${DEBIAN_VERSION} stable" >> /etc/apt/sources.list.d/docker.list
+mkdir -m 0755 -p /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  ${DEBIAN_VERSION} stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
 apt-get update
 
-PACKAGE_VERSION=$(apt-cache policy docker-ce | awk -v dv=${DOCKER_VERSION}~ '$1 ~ dv {print $1}')
+PACKAGE_VERSION=$(apt-cache policy docker-ce | awk -v dv=${DOCKER_VERSION} '$1 ~ dv {print $1}')
+
+ apt-get install -y \
+    docker-ce=${PACKAGE_VERSION} \
+    docker-ce-cli=${PACKAGE_VERSION} \
+    docker-buildx-plugin
 
-apt-get install -y docker-ce=${PACKAGE_VERSION} docker-ce-cli=${PACKAGE_VERSION}
 apt-get -yq autoremove
 apt-get clean -yqq
 rm -rf /var/lib/apt/lists/*
diff --git a/scripts/lib/custom-docker-build b/scripts/lib/custom-docker-build
index f9782cb..31a8072 100755
--- a/scripts/lib/custom-docker-build
+++ b/scripts/lib/custom-docker-build
@@ -173,27 +173,6 @@ function print_docker_args() {
     printf -- "--build-arg DOCKER_VERSION=%s " "$1"
 }
 
-function print_buildx_args() {
-    declare -A BUILDX_DOWNLOAD_SHA256
-
-    case "$1" in
-        0.8)
-            BUILDX_VERSION=0.8.2
-            BUILDX_DOWNLOAD_SHA256[amd64]=c64de4f3c30f7a73ff9db637660c7aa0f00234368105b0a09fa8e24eebe910c3
-            BUILDX_DOWNLOAD_SHA256[arm64]=304d3d9822c75f98ad9cf57f0c234bcf326bbb96d791d551728cadd72a7a377f
-            ;;
-        0.9)
-            BUILDX_VERSION=0.9.1
-            BUILDX_DOWNLOAD_SHA256[amd64]=a7fb95177792ca8ffc7243fad7bf2f33738b8b999a184b6201f002a63c43d136
-            BUILDX_DOWNLOAD_SHA256[arm64]=bbf6a76bf9aef9c5759ff225b97ce23a24fc11e4fa3cdcae36e5dcf1de2cffc5
-            ;;
-        *) fail "Unknown buildx version $1" ;;
-    esac
-
-    printf -- "--build-arg BUILDX_VERSION=%s " "$BUILDX_VERSION"
-    printf -- "--build-arg BUILDX_DOWNLOAD_SHA256=%q " "${BUILDX_DOWNLOAD_SHA256[*]}"
-}
-
 function print_graphicsmagick_args() {
     case "$1" in
         1.3.29)
@@ -374,7 +353,6 @@ function parse_arguments() {
                 RUST) print_rust_args $version ;;
                 CHROME) print_chrome_args $version ;;
                 DOCKER) print_docker_args $version ;;
-                BUILDX) print_buildx_args $version ;;
                 GIT) print_git_args $version ;;
                 LFS) print_lfs_args $version ;;
                 NODE) print_node_args $version ;;
diff --git a/scripts/lib/custom-docker.sh b/scripts/lib/custom-docker.sh
index a14a69d..0b9599e 100644
--- a/scripts/lib/custom-docker.sh
+++ b/scripts/lib/custom-docker.sh
@@ -2,7 +2,7 @@
 # various variable operations used in this script.
 
 PATH_TOOLS=(DEBIAN OS UBI RUBY GOLANG RUST NODE POSTGRESQL)
-TAG_TOOLS=(BUNDLER RUBYGEMS GIT LFS CHROME YARN GRAPHICSMAGICK BAZELISK DOCKER BUILDX GCLOUD KUBECTL HELM)
+TAG_TOOLS=(BUNDLER RUBYGEMS GIT LFS CHROME YARN GRAPHICSMAGICK BAZELISK DOCKER GCLOUD KUBECTL HELM)
 
 # Generate the docker image path using the components that were specified via
 # variables.