From 34ce2513c8a8de7a173e231a3c562cf5ddfa2fc3 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Tue, 27 Feb 2018 12:39:08 -0800 Subject: [PATCH] Add an image for git v2.16 and Chrome v63 Also adds a caching mechanism for storing the latest version in an S3 bucket. --- .gitlab-ci.yml | 13 ++++++++++++ README.md | 14 ++++++------- scripts/cache-google-chrome | 41 +++++++++++++++++++++++++++++++++++++ scripts/custom-docker-build | 2 +- scripts/install-chrome | 21 +++++++++++++++++-- 5 files changed, 80 insertions(+), 11 deletions(-) create mode 100755 scripts/cache-google-chrome diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 033e67c..fe125c7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,6 +57,7 @@ ruby-2.3-golang-1.10-git-2.9 test: *test_custom ruby-2.3-golang-1.10-git-2.16 test: *test_custom ruby-2.3.5-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom ruby-2.3.6-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom +ruby-2.3.6-golang-1.9-git-2.16-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom ruby-2.3.6-golang-1.9-git-2.16-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom golang-1.8.6-git-2.9.0 test: *test_custom golang-1.8.6-git-2.14 test: *test_custom @@ -92,6 +93,7 @@ ruby-2.3-golang-1.10-git-2.14: *build_and_deploy_custom ruby-2.3-golang-1.10-git-2.9: *build_and_deploy_custom ruby-2.3.5-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom ruby-2.3.6-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom +ruby-2.3.6-golang-1.9-git-2.16-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom ruby-2.3.6-golang-1.9-git-2.16-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom golang-1.8.6-git-2.9.0: *build_and_deploy_custom golang-1.8.6-git-2.14: *build_and_deploy_custom @@ -115,6 +117,17 @@ omnibus-gitlab-wheezy: *build_and_deploy omnibus-gitlab-xenial: *build_and_deploy sitespeed-gitlab: *build_and_deploy +cache-google-chrome: + stage: build + variables: + MOUNT_POINT: /builds/$CI_PROJECT_PATH/mnt + script: + - mkdir -p "$MOUNT_POINT" + - cp scripts/cache-google-chrome $MOUNT_POINT + - docker run -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" -v "$MOUNT_POINT:/mnt" ubuntu:16.04 /mnt/cache-google-chrome + only: + - master + # Old Google Chrome images are listed here for reference: # These still exist in our docker repisotiry, but they cannot be built anymore since diff --git a/README.md b/README.md index 0f301c7..0e129b9 100644 --- a/README.md +++ b/README.md @@ -63,14 +63,12 @@ the name would be `ruby-2.4-golang-1.9-git-2.14`. # Note regarding Google Chrome -Google has a policy of yanking older versions of Google Chrome from their PPA whenever a new major -version is released. This means images with non-current chrome versions will fail to build. Until -a better solution is worked out, we will remove images based on older Chrome versions from the build -process, but preserve them for reference within a comment in the `.gitlab-ci.yml` file. - -These images will persist in the docker repository for legacy use, but will no longer be recompiled. -If an update ever needs to be made to an image containing an older version of Chrome, a workaround -can be modeled after https://gitlab.com/gitlab-org/gitlab-build-images/merge_requests/54 +Google has a policy of yanking older versions of Google Chrome from their PPA +whenever a new major version is released. To help maintain consistant build +images, there is a CI step that saves the latest Google Chrome .deb into an S3 +bucket. The install for Chrome will attempt to retrieve from the bucket if it +cannot find the file in the apt repository. See `scripts/cache-google-chrome` +for more details. # Contributing diff --git a/scripts/cache-google-chrome b/scripts/cache-google-chrome new file mode 100755 index 0000000..ee287f9 --- /dev/null +++ b/scripts/cache-google-chrome @@ -0,0 +1,41 @@ +#!/bin/bash +# This script attempts to copy the latest version of the Google Chrome Debian +# package into our own S3 bucket. Google yanks old versions regularly, making +# it hard to keep up with all the new versions. + +set -e + +if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then + echo "AWS credentials are not defined, skipping this step" + exit 0 +fi + +CHROME_S3_BUCKET=${1:-gitlab-google-chrome-stable} + +export DEBIAN_FRONTEND=noninteractive +apt-get -y update +apt-get -y install apt-utils curl awscli + +curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - +echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list + +echo "Updating apt to get Google Chrome packages..." + +apt-get -y -q update + +echo "Checking for latest Chrome version in apt repository..." + +LATEST_VERSION=$(apt-cache show google-chrome-stable | grep Version | sort | tail -1 | sed -e "s/Version: //") +CHROME_DEB="google-chrome-stable_${LATEST_VERSION}_amd64.deb" +CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}" + +echo "Checking if cache has $CHROME_DEB" +FILE_CHECK=$(curl -sL -w "%{http_code}\\n" "$CHROME_URL" -o /dev/null) + +if [ "$FILE_CHECK" -eq "200" ]; then + echo "Latest version $LATEST_VERSION is already cached!" +else + apt-get -d -y install google-chrome-stable + echo "Transfering $CHROME_DEB to S3 cache" + aws s3 cp /var/cache/apt/archives/$CHROME_DEB s3://$CHROME_S3_BUCKET +fi diff --git a/scripts/custom-docker-build b/scripts/custom-docker-build index 4f709eb..5922de0 100755 --- a/scripts/custom-docker-build +++ b/scripts/custom-docker-build @@ -33,7 +33,7 @@ function print_chrome_args() { 60|60.0) CHROME_VERSION=60.0.3112.90-1 ;; 61|61.0) CHROME_VERSION=61.0.3163.100-1 ;; 62|62.0) CHROME_VERSION=62.0.3202.89-1 ;; - 63|63.0) CHROME_VERSION=63.0.3239.108-1 ;; + 63|63.0) CHROME_VERSION=63.0.3239.132-1 ;; 64|64.0) CHROME_VERSION=64.0.3282.186-1 ;; *) echo "Unknown chrome version $1"; exit 1; esac diff --git a/scripts/install-chrome b/scripts/install-chrome index fc5309c..5db385e 100755 --- a/scripts/install-chrome +++ b/scripts/install-chrome @@ -9,8 +9,25 @@ export DEBIAN_FRONTEND=noninteractive curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list -apt-get update -q -env DEBIAN_FRONTEND="noninteractive" apt-get install -y google-chrome-stable=$CHROME_VERSION +apt-get update -q -y + +# Download from our local S3 bucket if we can't find the package in the repository +echo "Searching for $CHROME_VERSION in apt repository" +CHECK_VERSION=$(apt-cache show google-chrome-stable | grep Version | grep "$CHROME_VERSION") || true + +if [[ -z $CHECK_VERSION ]]; then + CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb" + CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}" + echo "Downloading $CHROME_URL" + curl --silent --show-error --fail -O $CHROME_URL + dpkg -i ./$CHROME_DEB || true + apt-get install -f -y + rm -f $CHROME_DEB +else + echo "Installing via apt-get" + apt-get install -y google-chrome-stable=$CHROME_VERSION +fi + rm -rf /var/lib/apt/lists/* # Install ChromeDriver