Merge branch 'sh-build-chrome-from-cache' into 'master'

Add an image for git v2.16 and Chrome v63

Closes #17

See merge request gitlab-org/gitlab-build-images!83

.gitlab-ci.yml

@@ -1,5 +1,10 @@
 image: docker:git
 
+stages:
+  - build
+  - test
+  - automation
+
 services:
 - docker:dind
 
@@ -55,9 +60,8 @@ ruby-2.3-golang-1.9-git-2.9 test: *test_custom
 ruby-2.3-golang-1.10-git-2.14 test: *test_custom
 ruby-2.3-golang-1.10-git-2.9 test: *test_custom
 ruby-2.3-golang-1.10-git-2.16 test: *test_custom
-ruby-2.3.5-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom
-ruby-2.3.6-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom
-ruby-2.3.6-golang-1.9-git-2.16-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom
+ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom
+ruby-2.3.6-golang-1.9-git-2.16-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6 test: *test_custom
 golang-1.8.6-git-2.9.0 test: *test_custom
 golang-1.8.6-git-2.14 test: *test_custom
 golang-1.9-git-2.9.0 test: *test_custom
@@ -66,7 +70,7 @@ golang-1.9-git-2.16 test: *test_custom
 golang-1.10-git-2.9.0 test: *test_custom
 golang-1.10-git-2.14 test: *test_custom
 golang-1.10-git-2.16 test: *test_custom
-node-8.9-chrome-64.0-yarn-1.2 test: *test_custom
+node-8.9-chrome-63.0-yarn-1.2 test: *test_custom
 
 www-gitlab-com test: *test_build
 gitlab-qa test: *test_build
@@ -90,9 +94,8 @@ ruby-2.3-golang-1.9-git-2.9: *build_and_deploy_custom
 ruby-2.3-golang-1.10-git-2.16: *build_and_deploy_custom
 ruby-2.3-golang-1.10-git-2.14: *build_and_deploy_custom
 ruby-2.3-golang-1.10-git-2.9: *build_and_deploy_custom
-ruby-2.3.5-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom
-ruby-2.3.6-golang-1.9-git-2.14-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom
-ruby-2.3.6-golang-1.9-git-2.16-chrome-64.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom
+ruby-2.3.6-golang-1.9-git-2.14-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom
+ruby-2.3.6-golang-1.9-git-2.16-chrome-63.0-node-8.x-yarn-1.2-postgresql-9.6: *build_and_deploy_custom
 golang-1.8.6-git-2.9.0: *build_and_deploy_custom
 golang-1.8.6-git-2.14: *build_and_deploy_custom
 golang-1.9-git-2.9.0: *build_and_deploy_custom
@@ -101,7 +104,7 @@ golang-1.9-git-2.16: *build_and_deploy_custom
 golang-1.10-git-2.9.0: *build_and_deploy_custom
 golang-1.10-git-2.14: *build_and_deploy_custom
 golang-1.10-git-2.16: *build_and_deploy_custom
-node-8.9-chrome-64.0-yarn-1.2: *build_and_deploy_custom
+node-8.9-chrome-63.0-yarn-1.2: *build_and_deploy_custom
 
 www-gitlab-com: *build_and_deploy
 gitlab-qa: *build_and_deploy
@@ -115,6 +118,15 @@ omnibus-gitlab-wheezy: *build_and_deploy
 omnibus-gitlab-xenial: *build_and_deploy
 sitespeed-gitlab: *build_and_deploy
 
+cache-google-chrome:
+  stage: automation
+  variables:
+    MOUNT_POINT: /builds/$CI_PROJECT_PATH/mnt
+  script:
+    - mkdir -p "$MOUNT_POINT"
+    - cp scripts/cache-google-chrome $MOUNT_POINT
+    - docker run -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" -v "$MOUNT_POINT:/mnt" ubuntu:16.04 /mnt/cache-google-chrome
+
 # Old Google Chrome images are listed here for reference:
 
 # These still exist in our docker repisotiry, but they cannot be built anymore since

README.md

@@ -63,14 +63,12 @@ the name would be `ruby-2.4-golang-1.9-git-2.14`.
 
 # Note regarding Google Chrome
 
-Google has a policy of yanking older versions of Google Chrome from their PPA whenever a new major
-version is released.  This means images with non-current chrome versions will fail to build.  Until
-a better solution is worked out, we will remove images based on older Chrome versions from the build
-process, but preserve them for reference within a comment in the `.gitlab-ci.yml` file.
-
-These images will persist in the docker repository for legacy use, but will no longer be recompiled.
-If an update ever needs to be made to an image containing an older version of Chrome, a workaround
-can be modeled after https://gitlab.com/gitlab-org/gitlab-build-images/merge_requests/54
+Google has a policy of yanking older versions of Google Chrome from their PPA
+whenever a new major version is released. To help maintain consistant build
+images, there is a CI step that saves the latest Google Chrome .deb into an S3
+bucket.  The install for Chrome will attempt to retrieve from the bucket if it
+cannot find the file in the apt repository. See `scripts/cache-google-chrome`
+for more details.
 
 # Contributing
 

scripts/cache-google-chrome

@@ -0,0 +1,41 @@
+#!/bin/bash
+# This script attempts to copy the latest version of the Google Chrome Debian
+# package into our own S3 bucket. Google yanks old versions regularly, making
+# it hard to keep up with all the new versions.
+
+set -e
+
+if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then
+    echo "AWS credentials are not defined, skipping this step"
+    exit 0
+fi
+
+CHROME_S3_BUCKET=${1:-gitlab-google-chrome-stable}
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update
+apt-get -y install apt-utils curl awscli
+
+curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add -
+echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
+
+echo "Updating apt to get Google Chrome packages..."
+
+apt-get -y -q update
+
+echo "Checking for latest Chrome version in apt repository..."
+
+LATEST_VERSION=$(apt-cache show google-chrome-stable | grep Version | sort | tail -1 | sed -e "s/Version: //")
+CHROME_DEB="google-chrome-stable_${LATEST_VERSION}_amd64.deb"
+CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}"
+
+echo "Checking if cache has $CHROME_DEB"
+FILE_CHECK=$(curl -sL -I -w "%{http_code}\\n" "$CHROME_URL" -o /dev/null)
+
+if [ "$FILE_CHECK" -eq "200" ]; then
+    echo "Latest version $LATEST_VERSION is already cached!"
+else
+    apt-get -d -y install google-chrome-stable
+    echo "Transfering $CHROME_DEB to S3 cache"
+    aws s3 cp /var/cache/apt/archives/$CHROME_DEB s3://$CHROME_S3_BUCKET
+fi

scripts/custom-docker-build

@@ -33,7 +33,7 @@ function print_chrome_args() {
         60|60.0) CHROME_VERSION=60.0.3112.90-1 ;;
         61|61.0) CHROME_VERSION=61.0.3163.100-1 ;;
         62|62.0) CHROME_VERSION=62.0.3202.89-1 ;;
-        63|63.0) CHROME_VERSION=63.0.3239.108-1 ;;
+        63|63.0) CHROME_VERSION=63.0.3239.132-1 ;;
         64|64.0) CHROME_VERSION=64.0.3282.186-1 ;;
         *) echo "Unknown chrome version $1"; exit 1;
     esac

scripts/install-chrome

@@ -9,8 +9,25 @@ export DEBIAN_FRONTEND=noninteractive
 curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add -
 echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list
 
-apt-get update -q
-env DEBIAN_FRONTEND="noninteractive" apt-get install -y google-chrome-stable=$CHROME_VERSION
+apt-get update -q -y
+
+# Download from our local S3 bucket if we can't find the package in the repository
+echo "Searching for $CHROME_VERSION in apt repository"
+CHECK_VERSION=$(apt-cache show google-chrome-stable | grep Version | grep "$CHROME_VERSION") || true
+
+if [[ -z $CHECK_VERSION ]]; then
+    CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb"
+    CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}"
+    echo "Downloading $CHROME_URL"
+    curl --silent --show-error --fail -O $CHROME_URL
+    dpkg -i ./$CHROME_DEB || true
+    apt-get install -f -y
+    rm -f $CHROME_DEB
+else
+    echo "Installing via apt-get"
+    apt-get install -y google-chrome-stable=$CHROME_VERSION
+fi
+
 rm -rf /var/lib/apt/lists/*
 
 # Install ChromeDriver