From a9c5fa7b10238d68ee0cfd0a69e06545df7ac4f3 Mon Sep 17 00:00:00 2001 From: Lukas Eipert Date: Tue, 22 Feb 2022 10:00:31 +0100 Subject: [PATCH] Cache Google Chrome deb with generic packages rather than AWS We currently cache Google Chrome deb files in AWS. Since GitLab 14.8 Generic Packages have become generally available. So in order to remove the dependency on AWS and to dogfood the feature, we are going to cache the Google Chrome deb files with GitLab https://docs.gitlab.com/ee/user/packages/generic_packages/ --- .gitlab-ci.yml | 12 +++++------- README.md | 8 ++++---- scripts/cache-google-chrome | 38 +++++++++++++++++++++---------------- scripts/install-chrome | 9 ++++++--- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5e47b9..fbc0839 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,14 +23,12 @@ variables: DOCKER_HOST: tcp://docker:2375 cache-google-chrome: + image: ubuntu:16.04 stage: automation - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH # Starts the job immediately needs: [] - variables: - MOUNT_POINT: /builds/$CI_PROJECT_PATH/mnt script: - - mkdir -p "$MOUNT_POINT" - - cp scripts/cache-google-chrome $MOUNT_POINT - - docker run -e "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" -e "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" -v "$MOUNT_POINT:/mnt" ubuntu:16.04 /mnt/cache-google-chrome + - bash ./scripts/cache-google-chrome + # disable DinD + before_script: [] + services: [] diff --git a/README.md b/README.md index 1b1c916..3036baf 100644 --- a/README.md +++ b/README.md @@ -83,10 +83,10 @@ unexpectedly. For reference, this happened in the past: https://gitlab.com/gitla Google has a policy of yanking older versions of Google Chrome from their PPA whenever a new major version is released. To help maintain consistent build -images, there is a CI step that saves the latest Google Chrome .deb into an S3 -bucket. The install for Chrome will attempt to retrieve from the bucket if it -cannot find the file in the apt repository. See `scripts/cache-google-chrome` -for more details. +images, there is a CI step that saves the latest Google Chrome .deb into an the +GitLab package registry. The install for Chrome will attempt to retrieve from the +registry if it cannot find the file in the apt repository. +See `scripts/cache-google-chrome` for more details. ## Contributing diff --git a/scripts/cache-google-chrome b/scripts/cache-google-chrome index a1a3f8d..690d23d 100755 --- a/scripts/cache-google-chrome +++ b/scripts/cache-google-chrome @@ -1,20 +1,15 @@ #!/bin/bash # This script attempts to copy the latest version of the Google Chrome Debian -# package into our own S3 bucket. Google yanks old versions regularly, making -# it hard to keep up with all the new versions. +# package into our own package registry. Google yanks old versions regularly, +# making it hard to keep up with all the new versions. set -e -if [[ -z $AWS_ACCESS_KEY_ID || -z $AWS_SECRET_ACCESS_KEY ]]; then - echo "AWS credentials are not defined, skipping this step" - exit 0 -fi - -CHROME_S3_BUCKET=${1:-gitlab-google-chrome-stable} +PKG=google-chrome-stable export DEBIAN_FRONTEND=noninteractive apt-get -y update -apt-get -y install apt-utils curl awscli +apt-get -y install apt-utils curl bash curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google.list @@ -25,17 +20,28 @@ apt-get -y -q update echo "Checking for latest Chrome version in apt repository..." -LATEST_VERSION=$(apt-cache show google-chrome-stable | grep Version | sort | tail -1 | sed -e "s/Version: //") +LATEST_VERSION=$(apt-cache show $PKG | grep Version | sort | tail -1 | sed -e "s/Version: //") CHROME_DEB="google-chrome-stable_${LATEST_VERSION}_amd64.deb" -CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}" +CHROME_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${PKG}/${LATEST_VERSION}/${CHROME_DEB}" echo "Checking if cache has $CHROME_DEB" -FILE_CHECK=$(curl -sL -I -w "%{http_code}\\n" "$CHROME_URL" -o /dev/null) +FILE_CHECK=$(curl --silent --location --head --output /dev/null --write "%{http_code}\\n" "$CHROME_URL") if [ "$FILE_CHECK" -eq "200" ]; then - echo "Latest version $LATEST_VERSION is already cached!" + echo "Latest version $LATEST_VERSION is already cached!" else - apt-get -d -y install google-chrome-stable - echo "Transfering $CHROME_DEB to S3 cache" - aws s3 cp /var/cache/apt/archives/$CHROME_DEB s3://$CHROME_S3_BUCKET + echo "Downloading latest Chrome version ($LATEST_VERSION) in apt repository..." + cd /tmp + apt-get download $PKG + + if ! [ -f "$CHROME_DEB" ]; then + echo "Downloaded file didn't have expected name: $CHROME_DEB" + ls + exit 1 + fi + + echo "Transferring $CHROME_DEB to GitLab packages" + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \ + --upload-file "./{$CHROME_DEB}" \ + "$CHROME_URL" fi diff --git a/scripts/install-chrome b/scripts/install-chrome index bed2782..e2e0513 100755 --- a/scripts/install-chrome +++ b/scripts/install-chrome @@ -5,6 +5,9 @@ IFS=$'\n\t' CHROME_VERSION=${1:-97.0.4692.99-1} CHROME_DRIVER_VERSION=${2:-97.0.4692.71} +# We hard code the URL rather than using $CI_API_V4_URL $CI_PROJECT_ID, +# because we would need to forward those variables +CHROME_DOWNLOAD_URL_BASE="https://gitlab.com/api/v4/projects/1075790/packages/generic/google-chrome-stable" export DEBIAN_FRONTEND=noninteractive curl -sS -L https://dl.google.com/linux/linux_signing_key.pub | apt-key add - @@ -12,14 +15,14 @@ echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /et apt-get update -q -y -# Download from our local S3 bucket if we can't find the package in the repository +# Download from our package registry if we can't find the package in the apt repository echo "Searching for $CHROME_VERSION in apt repository" CHECK_VERSION=$(apt-cache show google-chrome-stable | grep Version | grep "$CHROME_VERSION") || true if [[ -z $CHECK_VERSION ]]; then CHROME_DEB="google-chrome-stable_${CHROME_VERSION}_amd64.deb" - CHROME_URL="https://s3.amazonaws.com/gitlab-google-chrome-stable/${CHROME_DEB}" - echo "Downloading $CHROME_URL" + CHROME_URL="${CHROME_DOWNLOAD_URL_BASE}/${CHROME_VERSION}/${CHROME_DEB}" + echo "Downloading from our Package registry: $CHROME_URL" curl --silent --show-error --fail -O $CHROME_URL dpkg -i ./$CHROME_DEB || true apt-get install -f -y